This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 7bdfbba56f HDDS-7984. [Snapshot] Store Ozone keys ever renamed in
renamedKeyTable for efficient lookup in SnapshotDeletingService (#4333)
7bdfbba56f is described below
commit 7bdfbba56f0f9b47d62b702e32ee846a2701b9fe
Author: Aswin Shakil Balasubramanian <[email protected]>
AuthorDate: Thu Mar 2 23:05:32 2023 -0800
HDDS-7984. [Snapshot] Store Ozone keys ever renamed in renamedKeyTable for
efficient lookup in SnapshotDeletingService (#4333)
---
.../hadoop/ozone/om/helpers/OmKeyRenameInfo.java | 91 ++++++++++++++++++++++
.../src/main/proto/OmClientProtocol.proto | 4 +
.../apache/hadoop/ozone/om/OMMetadataManager.java | 15 ++++
.../ozone/om/codec/OmKeyRenameInfoCodec.java | 55 +++++++++++++
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 38 +++++++--
.../hadoop/ozone/om/codec/OMDBDefinition.java | 12 ++-
.../ozone/om/response/key/OMKeyRenameResponse.java | 21 ++++-
.../response/key/OMKeyRenameResponseWithFSO.java | 19 ++++-
.../om/response/key/OMKeysRenameResponse.java | 22 +++++-
.../snapshot/OMSnapshotCreateResponse.java | 28 ++++++-
.../snapshot/TestOMSnapshotCreateRequest.java | 65 ++++++++++++++++
.../om/response/key/TestOMKeyRenameResponse.java | 22 +++++-
.../key/TestOMKeyRenameResponseWithFSO.java | 9 +++
13 files changed, 385 insertions(+), 16 deletions(-)
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyRenameInfo.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyRenameInfo.java
new file mode 100644
index 0000000000..bb6001d20c
--- /dev/null
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyRenameInfo.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.hadoop.ozone.om.helpers;
+
+
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyRenameInfo;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Argument for renamedKeyTable. Helps to store List<String> which represents
+ * all the renames that happened to particular key in between snapshots.
+ */
+public class OmKeyRenameInfo {
+ private List<String> keyNamesList;
+
+ public OmKeyRenameInfo(List<String> keyNamesList) {
+ this.keyNamesList = keyNamesList;
+ }
+
+ public OmKeyRenameInfo(String keyRenameInfo) {
+ this.keyNamesList = new ArrayList<>();
+ this.keyNamesList.add(keyRenameInfo);
+ }
+
+ public void addOmKeyRenameInfo(String keyRenameInfo) {
+ this.keyNamesList.add(keyRenameInfo);
+ }
+
+ public List<String> getOmKeyRenameInfoList() {
+ return keyNamesList;
+ }
+
+ public List<String> cloneOmKeyRenameInfoList() {
+ return new ArrayList<>(keyNamesList);
+ }
+
+
+ public static OmKeyRenameInfo getFromProto(KeyRenameInfo
+ keyRenameInfo) throws IOException {
+ List<String> list = new ArrayList<>(keyRenameInfo.getKeyNamesList());
+ return new OmKeyRenameInfo.Builder().setOmKeyRenameList(list).build();
+ }
+
+ public KeyRenameInfo getProto() {
+ List<String> list = new ArrayList<>(cloneOmKeyRenameInfoList());
+
+ KeyRenameInfo.Builder builder = KeyRenameInfo.newBuilder()
+ .addAllKeyNames(list);
+ return builder.build();
+ }
+
+ public OmKeyRenameInfo copyObject() {
+ return new OmKeyRenameInfo(new ArrayList<>(keyNamesList));
+ }
+
+ /**
+ * Builder of OmKeyRenameInfo.
+ */
+ public static class Builder {
+ private List<String> keyNamesList;
+
+ public Builder() { }
+
+ public OmKeyRenameInfo.Builder setOmKeyRenameList(List<String> stringList)
{
+ this.keyNamesList = stringList;
+ return this;
+ }
+
+ public OmKeyRenameInfo build() {
+ return new OmKeyRenameInfo(keyNamesList);
+ }
+ }
+
+}
diff --git
a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index a2b2994cf9..33ee575d6e 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -1028,6 +1028,10 @@ message RepeatedKeyInfo {
repeated KeyInfo keyInfo = 1;
}
+message KeyRenameInfo {
+ repeated string keyNames = 1;
+}
+
message OzoneFileStatusProto {
optional KeyInfo keyInfo = 2;
optional uint64 blockSize = 3;
diff --git
a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
index 0229e86dda..285fb0245b 100644
---
a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
+++
b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo;
import org.apache.hadoop.ozone.om.helpers.OmDBTenantState;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyRenameInfo;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock;
@@ -374,6 +375,8 @@ public interface OMMetadataManager extends DBStoreHAManager
{
Table<String, SnapshotInfo> getSnapshotInfoTable();
+ Table<String, OmKeyRenameInfo> getRenamedKeyTable();
+
/**
* Gets the OM Meta table.
* @return meta table reference.
@@ -482,6 +485,18 @@ public interface OMMetadataManager extends
DBStoreHAManager {
String getOpenFileName(long volumeId, long bucketId,
long parentObjectId, String fileName, long id);
+
+ /**
+ * Given a volume, bucket and a objectID, return the DB key name in
+ * renamedKeyTable.
+ *
+ * @param volume - volume name
+ * @param bucket - bucket name
+ * @param objectID - objectID of the key
+ * @return DB rename key as String.
+ */
+ String getRenameKey(String volume, String bucket, long objectID);
+
/**
* Returns the DB key name of a multipart upload key in OM metadata store.
*
diff --git
a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/OmKeyRenameInfoCodec.java
b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/OmKeyRenameInfoCodec.java
new file mode 100644
index 0000000000..dbd3b6f294
--- /dev/null
+++
b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/OmKeyRenameInfoCodec.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.hadoop.ozone.om.codec;
+
+import com.google.common.base.Preconditions;
+import com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.hadoop.hdds.utils.db.Codec;
+import org.apache.hadoop.ozone.om.helpers.OmKeyRenameInfo;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyRenameInfo;
+
+import java.io.IOException;
+
+/**
+ * Codec to encode OmKeyRenameInfo as byte array.
+ */
+public class OmKeyRenameInfoCodec implements Codec<OmKeyRenameInfo> {
+ @Override
+ public byte[] toPersistedFormat(OmKeyRenameInfo object) throws IOException {
+ Preconditions
+ .checkNotNull(object, "Null object can't be converted to byte array.");
+ return object.getProto().toByteArray();
+ }
+
+ @Override
+ public OmKeyRenameInfo fromPersistedFormat(byte[] rawData)
+ throws IOException {
+ Preconditions.checkNotNull(rawData,
+ "Null byte array can't converted to real object.");
+ try {
+ return OmKeyRenameInfo.getFromProto(KeyRenameInfo.parseFrom(rawData));
+ } catch (InvalidProtocolBufferException ex) {
+ throw new IllegalArgumentException(
+ "Can't encode the the raw data from the byte array", ex);
+ }
+ }
+
+ @Override
+ public OmKeyRenameInfo copyObject(OmKeyRenameInfo object) {
+ return object.copyObject();
+ }
+}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index 2a0241e3b3..ede9b5893a 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -64,6 +64,7 @@ import org.apache.hadoop.ozone.om.codec.OmPrefixInfoCodec;
import org.apache.hadoop.ozone.om.codec.OmDBTenantStateCodec;
import org.apache.hadoop.ozone.om.codec.OmVolumeArgsCodec;
import org.apache.hadoop.ozone.om.codec.RepeatedOmKeyInfoCodec;
+import org.apache.hadoop.ozone.om.codec.OmKeyRenameInfoCodec;
import org.apache.hadoop.ozone.om.codec.S3SecretValueCodec;
import org.apache.hadoop.ozone.om.codec.OmDBSnapshotInfoCodec;
import org.apache.hadoop.ozone.om.codec.TokenIdentifierCodec;
@@ -83,6 +84,7 @@ import org.apache.hadoop.ozone.om.helpers.OmDBTenantState;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyRenameInfo;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
@@ -193,6 +195,8 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
* |----------------------------------------------------------------------|
* | snapshotInfoTable | /volume/bucket/snapshotName -> SnapshotInfo |
* |----------------------------------------------------------------------|
+ * | renamedKeyTable | /volumeName/bucketName/objectID -> OmKeyRenameInfo|
+ * |----------------------------------------------------------------------|
*/
public static final String USER_TABLE = "userTable";
@@ -219,6 +223,7 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
"principalToAccessIdsTable";
public static final String TENANT_STATE_TABLE = "tenantStateTable";
public static final String SNAPSHOT_INFO_TABLE = "snapshotInfoTable";
+ public static final String RENAMED_KEY_TABLE = "renamedKeyTable";
static final String[] ALL_TABLES = new String[] {
USER_TABLE,
@@ -240,7 +245,8 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
TENANT_ACCESS_ID_TABLE,
PRINCIPAL_TO_ACCESS_IDS_TABLE,
TENANT_STATE_TABLE,
- SNAPSHOT_INFO_TABLE
+ SNAPSHOT_INFO_TABLE,
+ RENAMED_KEY_TABLE
};
private DBStore store;
@@ -267,7 +273,9 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
private Table tenantAccessIdTable;
private Table principalToAccessIdsTable;
private Table tenantStateTable;
+
private Table snapshotInfoTable;
+ private Table renamedKeyTable;
private boolean isRatisEnabled;
private boolean ignorePipelineinKey;
@@ -502,6 +510,7 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
.addTable(PRINCIPAL_TO_ACCESS_IDS_TABLE)
.addTable(TENANT_STATE_TABLE)
.addTable(SNAPSHOT_INFO_TABLE)
+ .addTable(RENAMED_KEY_TABLE)
.addCodec(OzoneTokenIdentifier.class, new TokenIdentifierCodec())
.addCodec(OmKeyInfo.class, new OmKeyInfoCodec(true))
.addCodec(RepeatedOmKeyInfo.class,
@@ -516,10 +525,9 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
.addCodec(OmDirectoryInfo.class, new OmDirectoryInfoCodec())
.addCodec(OmDBTenantState.class, new OmDBTenantStateCodec())
.addCodec(OmDBAccessIdInfo.class, new OmDBAccessIdInfoCodec())
- .addCodec(OmDBUserPrincipalInfo.class,
- new OmDBUserPrincipalInfoCodec())
- .addCodec(SnapshotInfo.class,
- new OmDBSnapshotInfoCodec());
+ .addCodec(OmDBUserPrincipalInfo.class, new
OmDBUserPrincipalInfoCodec())
+ .addCodec(SnapshotInfo.class, new OmDBSnapshotInfoCodec())
+ .addCodec(OmKeyRenameInfo.class, new OmKeyRenameInfoCodec());
}
/**
@@ -622,6 +630,11 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
String.class, SnapshotInfo.class);
checkTableStatus(snapshotInfoTable, SNAPSHOT_INFO_TABLE, addCacheMetrics);
+ // objectID -> renamedKeys (renamed keys for key table)
+ renamedKeyTable = this.store.getTable(RENAMED_KEY_TABLE,
+ String.class, OmKeyRenameInfo.class);
+ checkTableStatus(renamedKeyTable, RENAMED_KEY_TABLE, addCacheMetrics);
+ // TODO: [SNAPSHOT] Initialize table lock for renamedKeyTable.
}
/**
@@ -1614,6 +1627,11 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
return snapshotInfoTable;
}
+ @Override
+ public Table<String, OmKeyRenameInfo> getRenamedKeyTable() {
+ return renamedKeyTable;
+ }
+
/**
* Update store used by subclass.
*
@@ -1676,6 +1694,16 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
return openKey.toString();
}
+ @Override
+ public String getRenameKey(String volumeName, String bucketName,
+ long objectID) {
+ StringBuilder renameKey = new StringBuilder();
+ renameKey.append(OM_KEY_PREFIX).append(volumeName);
+ renameKey.append(OM_KEY_PREFIX).append(bucketName);
+ renameKey.append(OM_KEY_PREFIX).append(objectID);
+ return renameKey.toString();
+ }
+
@Override
public String getMultipartKey(long volumeId, long bucketId,
long parentID, String fileName,
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
index aa40cc87ad..ae888594b5 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyRenameInfo;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
@@ -229,6 +230,15 @@ public class OMDBDefinition implements DBDefinition {
SnapshotInfo.class,
new OmDBSnapshotInfoCodec());
+ public static final DBColumnFamilyDefinition<String, OmKeyRenameInfo>
+ RENAMED_KEY_TABLE =
+ new DBColumnFamilyDefinition<>(
+ OmMetadataManagerImpl.RENAMED_KEY_TABLE,
+ String.class, // /volumeName/bucketName/objectID
+ new StringCodec(),
+ OmKeyRenameInfo.class, // list of key renames
+ new OmKeyRenameInfoCodec());
+
@Override
public String getName() {
return OzoneConsts.OM_DB_NAME;
@@ -248,7 +258,7 @@ public class OMDBDefinition implements DBDefinition {
FILE_TABLE, OPEN_FILE_TABLE, DELETED_DIR_TABLE, META_TABLE,
TENANT_ACCESS_ID_TABLE,
PRINCIPAL_TO_ACCESS_IDS_TABLE, TENANT_STATE_TABLE,
- SNAPSHOT_INFO_TABLE};
+ SNAPSHOT_INFO_TABLE, RENAMED_KEY_TABLE};
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponse.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponse.java
index b5cc494f25..e755e75d62 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponse.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponse.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.ozone.om.response.key;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyRenameInfo;
import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.OMResponse;
@@ -30,11 +31,12 @@ import java.io.IOException;
import javax.annotation.Nonnull;
import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.KEY_TABLE;
+import static
org.apache.hadoop.ozone.om.OmMetadataManagerImpl.RENAMED_KEY_TABLE;
/**
* Response for RenameKey request.
*/
-@CleanupTableInfo(cleanupTables = {KEY_TABLE})
+@CleanupTableInfo(cleanupTables = {KEY_TABLE, RENAMED_KEY_TABLE})
public class OMKeyRenameResponse extends OmKeyResponse {
private String fromKeyName;
@@ -73,13 +75,26 @@ public class OMKeyRenameResponse extends OmKeyResponse {
BatchOperation batchOperation) throws IOException {
String volumeName = renameKeyInfo.getVolumeName();
String bucketName = renameKeyInfo.getBucketName();
+ String fromDbKey = omMetadataManager
+ .getOzoneKey(volumeName, bucketName, fromKeyName);
omMetadataManager.getKeyTable(getBucketLayout())
- .deleteWithBatch(batchOperation,
- omMetadataManager.getOzoneKey(volumeName, bucketName,
fromKeyName));
+ .deleteWithBatch(batchOperation, fromDbKey);
omMetadataManager.getKeyTable(getBucketLayout())
.putWithBatch(batchOperation,
omMetadataManager.getOzoneKey(volumeName, bucketName, toKeyName),
renameKeyInfo);
+
+ String renameDbKey = omMetadataManager.getRenameKey(
+ renameKeyInfo.getVolumeName(), renameKeyInfo.getBucketName(),
+ renameKeyInfo.getObjectID());
+ OmKeyRenameInfo omKeyRenameInfo = omMetadataManager.getRenamedKeyTable()
+ .get(renameDbKey);
+ if (omKeyRenameInfo == null) {
+ omKeyRenameInfo = new OmKeyRenameInfo(fromDbKey);
+ omMetadataManager.getRenamedKeyTable().putWithBatch(
+ batchOperation, renameDbKey, omKeyRenameInfo);
+ }
+
}
public OmKeyInfo getRenameKeyInfo() {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponseWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponseWithFSO.java
index 4b5f6d1fdb..eb8f4a2b6c 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponseWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponseWithFSO.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyRenameInfo;
import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
@@ -33,11 +34,13 @@ import java.io.IOException;
import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DIRECTORY_TABLE;
import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE;
+import static
org.apache.hadoop.ozone.om.OmMetadataManagerImpl.RENAMED_KEY_TABLE;
/**
* Response for RenameKey request - prefix layout.
*/
-@CleanupTableInfo(cleanupTables = {FILE_TABLE, DIRECTORY_TABLE})
+@CleanupTableInfo(cleanupTables = {FILE_TABLE, DIRECTORY_TABLE,
+ RENAMED_KEY_TABLE})
public class OMKeyRenameResponseWithFSO extends OMKeyRenameResponse {
private boolean isRenameDirectory;
@@ -88,7 +91,21 @@ public class OMKeyRenameResponseWithFSO extends
OMKeyRenameResponse {
.deleteWithBatch(batchOperation, getFromKeyName());
omMetadataManager.getKeyTable(getBucketLayout())
.putWithBatch(batchOperation, getToKeyName(), getRenameKeyInfo());
+
+ String renameDbKey = omMetadataManager.getRenameKey(
+ getRenameKeyInfo().getVolumeName(),
+ getRenameKeyInfo().getBucketName(),
+ getRenameKeyInfo().getObjectID());
+
+ OmKeyRenameInfo omKeyRenameInfo = omMetadataManager.getRenamedKeyTable()
+ .get(renameDbKey);
+ if (omKeyRenameInfo == null) {
+ omKeyRenameInfo = new OmKeyRenameInfo(getFromKeyName());
+ omMetadataManager.getRenamedKeyTable().putWithBatch(
+ batchOperation, renameDbKey, omKeyRenameInfo);
+ }
}
+
if (fromKeyParent != null) {
addDirToDBBatch(omMetadataManager, fromKeyParent,
volumeId, bucketId, batchOperation);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysRenameResponse.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysRenameResponse.java
index b09ff9e421..dd69092e52 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysRenameResponse.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysRenameResponse.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.hdds.utils.db.BatchOperation;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.helpers.OmRenameKeys;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyRenameInfo;
import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
@@ -31,13 +32,14 @@ import java.io.IOException;
import java.util.Map;
import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.KEY_TABLE;
+import static
org.apache.hadoop.ozone.om.OmMetadataManagerImpl.RENAMED_KEY_TABLE;
import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK;
import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.PARTIAL_RENAME;
/**
* Response for RenameKeys request.
*/
-@CleanupTableInfo(cleanupTables = {KEY_TABLE})
+@CleanupTableInfo(cleanupTables = {KEY_TABLE, RENAMED_KEY_TABLE})
public class OMKeysRenameResponse extends OMClientResponse {
private OmRenameKeys omRenameKeys;
@@ -77,14 +79,28 @@ public class OMKeysRenameResponse extends OMClientResponse {
String fromKeyName = entry.getKey();
OmKeyInfo newKeyInfo = entry.getValue();
String toKeyName = newKeyInfo.getKeyName();
+ String fromDbKey = omMetadataManager
+ .getOzoneKey(volumeName, bucketName, fromKeyName);
omMetadataManager.getKeyTable(getBucketLayout())
- .deleteWithBatch(batchOperation, omMetadataManager
- .getOzoneKey(volumeName, bucketName, fromKeyName));
+ .deleteWithBatch(batchOperation, fromDbKey);
omMetadataManager.getKeyTable(getBucketLayout())
.putWithBatch(batchOperation,
omMetadataManager.getOzoneKey(volumeName, bucketName, toKeyName),
newKeyInfo);
+
+
+ String renameDbKey = omMetadataManager.getRenameKey(
+ newKeyInfo.getVolumeName(), newKeyInfo.getBucketName(),
+ newKeyInfo.getObjectID());
+
+ OmKeyRenameInfo omKeyRenameInfo = omMetadataManager.getRenamedKeyTable()
+ .get(renameDbKey);
+ if (omKeyRenameInfo == null) {
+ omKeyRenameInfo = new OmKeyRenameInfo(fromDbKey);
+ omMetadataManager.getRenamedKeyTable().putWithBatch(
+ batchOperation, renameDbKey, omKeyRenameInfo);
+ }
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotCreateResponse.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotCreateResponse.java
index e5ba1c0511..718b67ec89 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotCreateResponse.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotCreateResponse.java
@@ -19,8 +19,11 @@
package org.apache.hadoop.ozone.om.response.snapshot;
import org.apache.hadoop.hdds.utils.db.BatchOperation;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OmSnapshotManager;
+import org.apache.hadoop.ozone.om.helpers.OmKeyRenameInfo;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
@@ -28,12 +31,15 @@ import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRespo
import javax.annotation.Nonnull;
import java.io.IOException;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
+import static
org.apache.hadoop.ozone.om.OmMetadataManagerImpl.RENAMED_KEY_TABLE;
import static
org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE;
/**
* Response for OMSnapshotCreateRequest.
*/
-@CleanupTableInfo(cleanupTables = {SNAPSHOT_INFO_TABLE})
+@CleanupTableInfo(cleanupTables = {SNAPSHOT_INFO_TABLE, RENAMED_KEY_TABLE})
public class OMSnapshotCreateResponse extends OMClientResponse {
public OMSnapshotCreateResponse(@Nonnull OMResponse omResponse,
@@ -68,5 +74,25 @@ public class OMSnapshotCreateResponse extends
OMClientResponse {
// Add to db
omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation,
key, snapshotInfo);
+
+ // TODO: [SNAPSHOT] Move to createOmSnapshotCheckpoint and add table lock
+ // Remove all entries from renamedKeyTable
+ TableIterator<String, ? extends Table.KeyValue<String, OmKeyRenameInfo>>
+ iterator = omMetadataManager.getRenamedKeyTable().iterator();
+
+ String dbSnapshotBucketKey = omMetadataManager.getBucketKey(
+ snapshotInfo.getVolumeName(), snapshotInfo.getBucketName())
+ + OM_KEY_PREFIX;
+ iterator.seek(dbSnapshotBucketKey);
+
+ while (iterator.hasNext()) {
+ String renameDbKey = iterator.next().getKey();
+ if (!renameDbKey.startsWith(dbSnapshotBucketKey)) {
+ break;
+ }
+ omMetadataManager.getRenamedKeyTable()
+ .deleteWithBatch(batchOperation, renameDbKey);
+ }
+
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java
index c029ea39e3..94e551a335 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java
@@ -23,6 +23,8 @@ package org.apache.hadoop.ozone.om.request.snapshot;
import java.util.UUID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.utils.db.BatchOperation;
import org.apache.hadoop.ozone.audit.AuditLogger;
import org.apache.hadoop.ozone.audit.AuditMessage;
@@ -32,8 +34,10 @@ import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import org.apache.hadoop.ozone.om.response.key.OMKeyRenameResponse;
import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager;
import org.apache.ozone.test.LambdaTestUtils;
import org.junit.After;
@@ -67,6 +71,7 @@ public class TestOMSnapshotCreateRequest {
private OzoneManager ozoneManager;
private OMMetrics omMetrics;
private OMMetadataManager omMetadataManager;
+ private BatchOperation batchOperation;
private String volumeName;
private String bucketName;
@@ -98,6 +103,7 @@ public class TestOMSnapshotCreateRequest {
AuditLogger auditLogger = mock(AuditLogger.class);
when(ozoneManager.getAuditLogger()).thenReturn(auditLogger);
Mockito.doNothing().when(auditLogger).logWrite(any(AuditMessage.class));
+ batchOperation = omMetadataManager.getStore().initBatchOperation();
volumeName = UUID.randomUUID().toString();
bucketName = UUID.randomUUID().toString();
@@ -111,6 +117,9 @@ public class TestOMSnapshotCreateRequest {
public void stop() {
omMetrics.unRegister();
Mockito.framework().clearInlineMocks();
+ if (batchOperation != null) {
+ batchOperation.close();
+ }
}
@Test
@@ -238,6 +247,50 @@ public class TestOMSnapshotCreateRequest {
omResponse.getStatus());
}
+ @Test
+ public void testEmptyRenamedKeyTable() throws Exception {
+ when(ozoneManager.isAdmin(any())).thenReturn(true);
+ OmKeyInfo toKeyInfo = addKey("key1");
+ OmKeyInfo fromKeyInfo = addKey("key2");
+
+ OMResponse omResponse =
+ OMResponse.newBuilder().setRenameKeyResponse(
+ OzoneManagerProtocolProtos.RenameKeyResponse.getDefaultInstance())
+ .setStatus(OzoneManagerProtocolProtos.Status.OK)
+ .setCmdType(OzoneManagerProtocolProtos.Type.RenameKey)
+ .build();
+ OMKeyRenameResponse omKeyRenameResponse =
+ new OMKeyRenameResponse(omResponse, fromKeyInfo.getKeyName(),
+ toKeyInfo.getKeyName(), toKeyInfo);
+
+ Assert.assertTrue(omMetadataManager.getRenamedKeyTable().isEmpty());
+ omKeyRenameResponse.addToDBBatch(omMetadataManager, batchOperation);
+ omMetadataManager.getStore().commitBatchOperation(batchOperation);
+ Assert.assertFalse(omMetadataManager.getRenamedKeyTable().isEmpty());
+
+ OMRequest omRequest =
+ OMRequestTestUtils.createSnapshotRequest(
+ volumeName, bucketName, snapshotName);
+ OMSnapshotCreateRequest omSnapshotCreateRequest = doPreExecute(omRequest);
+ String key = SnapshotInfo.getTableKey(volumeName,
+ bucketName, snapshotName);
+
+ Assert.assertNull(omMetadataManager.getSnapshotInfoTable().get(key));
+
+ //create entry
+ OMClientResponse omClientResponse =
+ omSnapshotCreateRequest.validateAndUpdateCache(ozoneManager, 1,
+ ozoneManagerDoubleBufferHelper);
+ omClientResponse.checkAndUpdateDB(omMetadataManager, batchOperation);
+ omMetadataManager.getStore().commitBatchOperation(batchOperation);
+
+ SnapshotInfo snapshotInfo =
+ omMetadataManager.getSnapshotInfoTable().get(key);
+ Assert.assertNotNull(snapshotInfo);
+ Assert.assertTrue(omMetadataManager.getRenamedKeyTable().isEmpty());
+
+ }
+
@Test
public void testEntryExists() throws Exception {
when(ozoneManager.isAdmin(any())).thenReturn(true);
@@ -290,4 +343,16 @@ public class TestOMSnapshotCreateRequest {
return new OMSnapshotCreateRequest(modifiedRequest);
}
+ private OmKeyInfo addKey(String keyName) {
+ return OMRequestTestUtils.createOmKeyInfo(volumeName, bucketName, keyName,
+ HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.ONE,
0L);
+ }
+
+ protected String addKeyToTable(OmKeyInfo keyInfo) throws Exception {
+ OMRequestTestUtils.addKeyToTable(false, false, keyInfo, 0, 0L,
+ omMetadataManager);
+ return omMetadataManager.getOzoneKey(keyInfo.getVolumeName(),
+ keyInfo.getBucketName(), keyInfo.getKeyName());
+ }
+
}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponse.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponse.java
index ecd80c897d..1e1294168c 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponse.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponse.java
@@ -24,6 +24,7 @@ import java.util.UUID;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyRenameInfo;
import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
import org.junit.Assert;
import org.junit.Test;
@@ -52,7 +53,7 @@ public class TestOMKeyRenameResponse extends
TestOMKeyResponse {
String toKeyName = UUID.randomUUID().toString();
OmKeyInfo toKeyInfo = getOmKeyInfo(toKeyName);
- OmKeyInfo fromKeyInfo = getOmKeyInfo(keyName);
+ OmKeyInfo fromKeyInfo = getOmKeyInfo(toKeyInfo, keyName);
String dbFromKey = addKeyToTable(fromKeyInfo);
String dbToKey = getDBKeyName(toKeyInfo);
@@ -63,6 +64,7 @@ public class TestOMKeyRenameResponse extends
TestOMKeyResponse {
.isExist(dbFromKey));
Assert.assertFalse(omMetadataManager.getKeyTable(getBucketLayout())
.isExist(dbToKey));
+ Assert.assertTrue(omMetadataManager.getRenamedKeyTable().isEmpty());
if (getBucketLayout() == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
Assert.assertFalse(omMetadataManager.getDirectoryTable()
.isExist(getDBKeyName(fromKeyParent)));
@@ -81,6 +83,17 @@ public class TestOMKeyRenameResponse extends
TestOMKeyResponse {
.isExist(dbFromKey));
Assert.assertTrue(omMetadataManager.getKeyTable(getBucketLayout())
.isExist(dbToKey));
+
+ String renameDbKey = omMetadataManager.getRenameKey(
+ fromKeyInfo.getVolumeName(), fromKeyInfo.getBucketName(),
+ fromKeyInfo.getObjectID());
+ Assert.assertTrue(omMetadataManager.getRenamedKeyTable()
+ .isExist(renameDbKey));
+
+ OmKeyRenameInfo omKeyRenameInfo =
+ omMetadataManager.getRenamedKeyTable().get(renameDbKey);
+ Assert.assertTrue(omKeyRenameInfo.getOmKeyRenameInfoList()
+ .contains(dbFromKey));
if (getBucketLayout() == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
Assert.assertTrue(omMetadataManager.getDirectoryTable()
.isExist(getDBKeyName(fromKeyParent)));
@@ -104,7 +117,7 @@ public class TestOMKeyRenameResponse extends
TestOMKeyResponse {
String toKeyName = UUID.randomUUID().toString();
OmKeyInfo toKeyInfo = getOmKeyInfo(toKeyName);
- OmKeyInfo fromKeyInfo = getOmKeyInfo(keyName);
+ OmKeyInfo fromKeyInfo = getOmKeyInfo(toKeyInfo, keyName);
String dbFromKey = addKeyToTable(fromKeyInfo);
String dbToKey = getDBKeyName(toKeyInfo);
@@ -145,6 +158,11 @@ public class TestOMKeyRenameResponse extends
TestOMKeyResponse {
replicationType, replicationFactor, 0L);
}
+ protected OmKeyInfo getOmKeyInfo(OmKeyInfo toKeyInfo,
+ String keyName) {
+ return getOmKeyInfo(keyName);
+ }
+
protected String addKeyToTable(OmKeyInfo keyInfo) throws Exception {
OMRequestTestUtils.addKeyToTable(false, false, keyInfo, clientID, 0L,
omMetadataManager);
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponseWithFSO.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponseWithFSO.java
index 6dcf38df9c..f2f9ccaf87 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponseWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponseWithFSO.java
@@ -44,6 +44,15 @@ public class TestOMKeyRenameResponseWithFSO extends
TestOMKeyRenameResponse {
bucketId + 100L, bucketId + 101L, 0L, Time.now());
}
+ @Override
+ protected OmKeyInfo getOmKeyInfo(OmKeyInfo toKeyInfo,
+ String keyName) {
+ return OMRequestTestUtils.createOmKeyInfo(toKeyInfo.getVolumeName(),
+ toKeyInfo.getBucketName(), keyName, replicationType,
+ replicationFactor, toKeyInfo.getObjectID(),
+ toKeyInfo.getParentObjectID(), 0L, toKeyInfo.getCreationTime());
+ }
+
@Override
protected String addKeyToTable(OmKeyInfo keyInfo) throws Exception {
OMRequestTestUtils.addFileToKeyTable(false, false,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
