This is an automated email from the ASF dual-hosted git repository.
xyao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new d34ab29 HDDS-3903. OzoneRpcClient support batch rename keys. (#1150)
d34ab29 is described below
commit d34ab29b622168c57bd2d3049d060c42735de7f4
Author: micah zhao <[email protected]>
AuthorDate: Sat Aug 29 04:54:24 2020 +0800
HDDS-3903. OzoneRpcClient support batch rename keys. (#1150)
---
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 +
.../apache/hadoop/ozone/client/OzoneBucket.java | 10 +
.../ozone/client/protocol/ClientProtocol.java | 12 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 13 +
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 1 +
.../org/apache/hadoop/ozone/audit/OMAction.java | 1 +
.../hadoop/ozone/om/exceptions/OMException.java | 5 +-
.../hadoop/ozone/om/helpers/OmRenameKeys.java | 59 +++++
.../ozone/om/protocol/OzoneManagerProtocol.java | 9 +
...OzoneManagerProtocolClientSideTranslatorPB.java | 32 +++
.../client/rpc/TestOzoneRpcClientAbstract.java | 106 +++++++-
.../src/main/proto/OmClientProtocol.proto | 26 ++
.../org/apache/hadoop/ozone/om/OzoneManager.java | 12 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 3 +
.../ozone/om/request/key/OMKeysRenameRequest.java | 271 +++++++++++++++++++++
.../om/response/key/OMKeysRenameResponse.java | 80 ++++++
.../om/request/key/TestOMKeysRenameRequest.java | 160 ++++++++++++
.../om/response/key/TestOMKeysRenameResponse.java | 131 ++++++++++
18 files changed, 919 insertions(+), 14 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 4b38094..9854d40 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -293,6 +293,8 @@ public final class OzoneConsts {
public static final String MAX_PARTS = "maxParts";
public static final String S3_BUCKET = "s3Bucket";
public static final String S3_GETSECRET_USER = "S3GetSecretUser";
+ public static final String RENAMED_KEYS_MAP = "renamedKeysMap";
+ public static final String UNRENAMED_KEYS_MAP = "unRenamedKeysMap";
public static final String MULTIPART_UPLOAD_PART_NUMBER = "partNumber";
public static final String MULTIPART_UPLOAD_PART_NAME = "partName";
public static final String BUCKET_ENCRYPTION_KEY = "bucketEncryptionKey";
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
index 79712bb..d71e03c 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
@@ -473,6 +473,16 @@ public class OzoneBucket extends WithMetadata {
}
/**
+ * Rename the key by keyMap, The key is fromKeyName and value is toKeyName.
+ * @param keyMap The key is original key name nad value is new key name.
+ * @throws IOException
+ */
+ public void renameKeys(Map<String, String> keyMap)
+ throws IOException {
+ proxy.renameKeys(volumeName, name, keyMap);
+ }
+
+ /**
* Initiate multipart upload for a specified key.
* @param keyName
* @param type
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index 9c662ef..1b8d93a 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -314,7 +314,17 @@ public interface ClientProtocol {
* @throws IOException
*/
void renameKey(String volumeName, String bucketName, String fromKeyName,
- String toKeyName) throws IOException;
+ String toKeyName) throws IOException;
+
+ /**
+ * Renames existing keys within a bucket.
+ * @param volumeName Name of the Volume
+ * @param bucketName Name of the Bucket
+ * @param keyMap The key is original key name nad value is new key name.
+ * @throws IOException
+ */
+ void renameKeys(String volumeName, String bucketName,
+ Map<String, String> keyMap) throws IOException;
/**
* Returns list of Keys in {Volume/Bucket} that matches the keyPrefix,
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index dc37f09..3f984d2 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -85,6 +85,7 @@ import
org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteList;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts;
import org.apache.hadoop.ozone.om.helpers.OmPartInfo;
+import org.apache.hadoop.ozone.om.helpers.OmRenameKeys;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil;
@@ -761,6 +762,18 @@ public class RpcClient implements ClientProtocol {
}
@Override
+ public void renameKeys(String volumeName, String bucketName,
+ Map<String, String> keyMap) throws IOException {
+ verifyVolumeName(volumeName);
+ verifyBucketName(bucketName);
+ HddsClientUtils.checkNotNull(keyMap);
+ OmRenameKeys omRenameKeys =
+ new OmRenameKeys(volumeName, bucketName, keyMap, null);
+ ozoneManagerClient.renameKeys(omRenameKeys);
+ }
+
+
+ @Override
public List<OzoneKey> listKeys(String volumeName, String bucketName,
String keyPrefix, String prevKey,
int maxListResult)
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
index bb9aec4..93e0e7f 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -246,6 +246,7 @@ public final class OmUtils {
case DeleteBucket:
case CreateKey:
case RenameKey:
+ case RenameKeys:
case DeleteKey:
case DeleteKeys:
case CommitKey:
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
index 6b34e81..3480063 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
@@ -32,6 +32,7 @@ public enum OMAction implements AuditAction {
DELETE_BUCKET,
DELETE_KEY,
RENAME_KEY,
+ RENAME_KEYS,
SET_OWNER,
SET_QUOTA,
UPDATE_VOLUME,
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
index 54b5458..e08dccb 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
@@ -227,6 +227,9 @@ public class OMException extends IOException {
DETECTED_LOOP_IN_BUCKET_LINKS,
- NOT_SUPPORTED_OPERATION
+ NOT_SUPPORTED_OPERATION,
+
+ PARTIAL_RENAME
+
}
}
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmRenameKeys.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmRenameKeys.java
new file mode 100644
index 0000000..d550817
--- /dev/null
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmRenameKeys.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.helpers;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * This class is used for rename keys.
+ */
+public class OmRenameKeys {
+
+ private String volume;
+ private String bucket;
+ private Map<String, String> fromAndToKey = new HashMap<>();
+ private Map<String, OmKeyInfo> fromKeyAndToKeyInfo = new HashMap<>();
+
+ public OmRenameKeys(String volume, String bucket,
+ Map<String, String> fromAndToKey,
+ Map<String, OmKeyInfo> fromKeyAndToKeyInfo) {
+ this.volume = volume;
+ this.bucket = bucket;
+ this.fromAndToKey = fromAndToKey;
+ this.fromKeyAndToKeyInfo = fromKeyAndToKeyInfo;
+ }
+
+ public String getVolume() {
+ return volume;
+ }
+
+ public String getBucket() {
+ return bucket;
+ }
+
+ public Map<String, String> getFromAndToKey() {
+ return fromAndToKey;
+ }
+
+ public Map<String, OmKeyInfo> getFromKeyAndToKeyInfo() {
+ return fromKeyAndToKeyInfo;
+ }
+
+}
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
index 9ae107b..267ac89 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -39,6 +39,7 @@ import
org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteList;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts;
+import org.apache.hadoop.ozone.om.helpers.OmRenameKeys;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
@@ -218,6 +219,14 @@ public interface OzoneManagerProtocol
void renameKey(OmKeyArgs args, String toKeyName) throws IOException;
/**
+ * Rename existing keys within a bucket.
+ * @param omRenameKeys Includes volume, bucket, and fromKey toKey name map
+ * and fromKey name toKey info Map.
+ * @throws IOException
+ */
+ void renameKeys(OmRenameKeys omRenameKeys) throws IOException;
+
+ /**
* Deletes an existing key.
*
* @param args the args of the key.
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index 1377c53..a6ea042 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.stream.Collectors;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
@@ -44,6 +45,7 @@ import
org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteList;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts;
+import org.apache.hadoop.ozone.om.helpers.OmRenameKeys;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
@@ -121,7 +123,10 @@ import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Recover
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RecoverTrashResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RemoveAclRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RemoveAclResponse;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeysArgs;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeysMap;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeyRequest;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeysRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenewDelegationTokenResponseProto;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListResponse;
@@ -676,6 +681,33 @@ public final class
OzoneManagerProtocolClientSideTranslatorPB
}
@Override
+ public void renameKeys(OmRenameKeys omRenameKeys) throws IOException {
+
+ List<RenameKeysMap> renameKeyList = new ArrayList<>();
+ for (Map.Entry< String, String> entry :
+ omRenameKeys.getFromAndToKey().entrySet()) {
+ RenameKeysMap.Builder renameKey = RenameKeysMap.newBuilder()
+ .setFromKeyName(entry.getKey())
+ .setToKeyName(entry.getValue());
+ renameKeyList.add(renameKey.build());
+ }
+
+ RenameKeysArgs.Builder renameKeyArgs = RenameKeysArgs.newBuilder()
+ .setVolumeName(omRenameKeys.getVolume())
+ .setBucketName(omRenameKeys.getBucket())
+ .addAllRenameKeysMap(renameKeyList);
+
+ RenameKeysRequest.Builder reqKeys = RenameKeysRequest.newBuilder()
+ .setRenameKeysArgs(renameKeyArgs.build());
+
+ OMRequest omRequest = createOMRequest(Type.RenameKeys)
+ .setRenameKeysRequest(reqKeys.build())
+ .build();
+
+ handleError(submitRequest(omRequest));
+ }
+
+ @Override
public void renameKey(OmKeyArgs args, String toKeyName) throws IOException {
RenameKeyRequest.Builder req = RenameKeyRequest.newBuilder();
KeyArgs keyArgs = KeyArgs.newBuilder()
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
index 32b6bca..45d07b0 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
@@ -110,6 +110,8 @@ import static
org.apache.hadoop.hdds.client.ReplicationType.STAND_ALONE;
import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS;
import static org.apache.hadoop.ozone.OzoneAcl.AclScope.DEFAULT;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR;
+import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
+import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.PARTIAL_RENAME;
import static
org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType.GROUP;
import static
org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType.USER;
import static
org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.READ;
@@ -121,6 +123,7 @@ import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+
import org.junit.Test;
/**
@@ -1202,7 +1205,7 @@ public abstract class TestOzoneRpcClientAbstract {
Assert.assertEquals(keyName, key.getName());
bucket.deleteKey(keyName);
- OzoneTestUtils.expectOmException(ResultCodes.KEY_NOT_FOUND,
+ OzoneTestUtils.expectOmException(KEY_NOT_FOUND,
() -> bucket.getKey(keyName));
}
@@ -1217,13 +1220,7 @@ public abstract class TestOzoneRpcClientAbstract {
OzoneVolume volume = store.getVolume(volumeName);
volume.createBucket(bucketName);
OzoneBucket bucket = volume.getBucket(bucketName);
- OzoneOutputStream out = bucket.createKey(fromKeyName,
- value.getBytes().length, STAND_ALONE,
- ONE, new HashMap<>());
- out.write(value.getBytes());
- out.close();
- OzoneKey key = bucket.getKey(fromKeyName);
- Assert.assertEquals(fromKeyName, key.getName());
+ createTestKey(bucket, fromKeyName, value);
// Rename to empty string should fail.
OMException oe = null;
@@ -1244,13 +1241,80 @@ public abstract class TestOzoneRpcClientAbstract {
} catch (OMException e) {
oe = e;
}
- Assert.assertEquals(ResultCodes.KEY_NOT_FOUND, oe.getResult());
+ Assert.assertEquals(KEY_NOT_FOUND, oe.getResult());
- key = bucket.getKey(toKeyName);
+ OzoneKey key = bucket.getKey(toKeyName);
Assert.assertEquals(toKeyName, key.getName());
}
@Test
+ public void testKeysRename() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String keyName1 = "dir/file1";
+ String keyName2 = "dir/file2";
+
+ String newKeyName1 = "dir/key1";
+ String newKeyName2 = "dir/key2";
+
+ String value = "sample value";
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+ createTestKey(bucket, keyName1, value);
+ createTestKey(bucket, keyName2, value);
+
+ Map<String, String> keyMap = new HashMap();
+ keyMap.put(keyName1, newKeyName1);
+ keyMap.put(keyName2, newKeyName2);
+ bucket.renameKeys(keyMap);
+
+ // new key should exist
+ Assert.assertEquals(newKeyName1, bucket.getKey(newKeyName1).getName());
+ Assert.assertEquals(newKeyName2, bucket.getKey(newKeyName2).getName());
+
+ // old key should not exist
+ assertKeyRenamedEx(bucket, keyName1);
+ assertKeyRenamedEx(bucket, keyName2);
+ }
+
+ @Test
+ public void testKeysRenameFail() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String keyName1 = "dir/file1";
+ String keyName2 = "dir/file2";
+
+ String newKeyName1 = "dir/key1";
+ String newKeyName2 = "dir/key2";
+
+ String value = "sample value";
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ // Create only keyName1 to test the partial failure of renameKeys.
+ createTestKey(bucket, keyName1, value);
+
+ Map<String, String> keyMap = new HashMap();
+ keyMap.put(keyName1, newKeyName1);
+ keyMap.put(keyName2, newKeyName2);
+
+ try {
+ bucket.renameKeys(keyMap);
+ } catch (OMException ex) {
+ Assert.assertEquals(PARTIAL_RENAME, ex.getResult());
+ }
+
+ // newKeyName1 should exist
+ Assert.assertEquals(newKeyName1, bucket.getKey(newKeyName1).getName());
+ // newKeyName2 should not exist
+ assertKeyRenamedEx(bucket, keyName2);
+ }
+
+ @Test
public void testListVolume() throws IOException {
String volBase = "vol-" + RandomStringUtils.randomNumeric(3);
//Create 10 volume vol-<random>-a-0-<random> to vol-<random>-a-9-<random>
@@ -2685,6 +2749,28 @@ public abstract class TestOzoneRpcClientAbstract {
Assert.assertNotNull(omMultipartUploadCompleteInfo.getHash());
}
+ private void createTestKey(OzoneBucket bucket, String keyName,
+ String keyValue) throws IOException {
+ OzoneOutputStream out = bucket.createKey(keyName,
+ keyValue.getBytes().length, STAND_ALONE,
+ ONE, new HashMap<>());
+ out.write(keyValue.getBytes());
+ out.close();
+ OzoneKey key = bucket.getKey(keyName);
+ Assert.assertEquals(keyName, key.getName());
+ }
+
+ private void assertKeyRenamedEx(OzoneBucket bucket, String keyName)
+ throws Exception {
+ OMException oe = null;
+ try {
+ bucket.getKey(keyName);
+ } catch (OMException e) {
+ oe = e;
+ }
+ Assert.assertEquals(KEY_NOT_FOUND, oe.getResult());
+ }
+
/**
* Tests GDPR encryption/decryption.
* 1. Create GDPR Enabled bucket.
diff --git
a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 1b2075e..c6e2949 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -60,6 +60,7 @@ enum Type {
CommitKey = 36;
AllocateBlock = 37;
DeleteKeys = 38;
+ RenameKeys = 39;
InitiateMultiPartUpload = 45;
CommitMultiPartUpload = 46;
@@ -126,6 +127,7 @@ message OMRequest {
optional CommitKeyRequest commitKeyRequest =
36;
optional AllocateBlockRequest allocateBlockRequest =
37;
optional DeleteKeysRequest deleteKeysRequest =
38;
+ optional RenameKeysRequest renameKeysRequest =
39;
optional MultipartInfoInitiateRequest initiateMultiPartUploadRequest =
45;
optional MultipartCommitUploadPartRequest commitMultiPartUploadRequest =
46;
@@ -198,6 +200,7 @@ message OMResponse {
optional CommitKeyResponse commitKeyResponse =
36;
optional AllocateBlockResponse allocateBlockResponse =
37;
optional DeleteKeysResponse deleteKeysResponse =
38;
+ optional RenameKeysResponse renameKeysResponse =
39;
optional MultipartInfoInitiateResponse initiateMultiPartUploadResponse =
45;
optional MultipartCommitUploadPartResponse commitMultiPartUploadResponse =
46;
@@ -308,6 +311,9 @@ enum Status {
DETECTED_LOOP_IN_BUCKET_LINKS = 63;
NOT_SUPPORTED_OPERATION = 64;
+
+ PARTIAL_RENAME = 65;
+
}
/**
@@ -839,6 +845,26 @@ message LookupKeyResponse {
optional uint64 openVersion = 4;
}
+message RenameKeysRequest {
+ required RenameKeysArgs renameKeysArgs = 1;
+}
+
+message RenameKeysArgs {
+ required string volumeName = 1;
+ required string bucketName = 2;
+ repeated RenameKeysMap renameKeysMap = 3;
+}
+
+message RenameKeysMap {
+ required string fromKeyName = 1;
+ required string toKeyName = 2;
+}
+
+message RenameKeysResponse{
+ repeated RenameKeysMap unRenamedKeys = 1;
+ optional bool status = 2;
+}
+
message RenameKeyRequest{
required KeyArgs keyArgs = 1;
required string toKeyName = 2;
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 13e47a7..9daeec1 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -48,7 +48,6 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import com.google.common.base.Optional;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
@@ -125,6 +124,7 @@ import
org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteList;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts;
+import org.apache.hadoop.ozone.om.helpers.OmRenameKeys;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
@@ -182,7 +182,7 @@ import com.google.common.base.Preconditions;
import com.google.protobuf.BlockingService;
import com.google.protobuf.ProtocolMessageEnum;
import org.apache.commons.lang3.StringUtils;
-
+import org.apache.commons.lang3.tuple.Pair;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED_DEFAULT;
@@ -2244,6 +2244,14 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
}
}
+
+ @Override
+ public void renameKeys(OmRenameKeys omRenameKeys)
+ throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented. As write requests use a new approach");
+ }
+
@Override
public void renameKey(OmKeyArgs args, String toKeyName) throws IOException {
Preconditions.checkNotNull(args);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
index ddb6841..681c0da 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
@@ -41,6 +41,7 @@ import
org.apache.hadoop.ozone.om.request.key.OMKeyCreateRequest;
import org.apache.hadoop.ozone.om.request.key.OMKeyDeleteRequest;
import org.apache.hadoop.ozone.om.request.key.OMKeyPurgeRequest;
import org.apache.hadoop.ozone.om.request.key.OMKeyRenameRequest;
+import org.apache.hadoop.ozone.om.request.key.OMKeysRenameRequest;
import org.apache.hadoop.ozone.om.request.key.OMTrashRecoverRequest;
import org.apache.hadoop.ozone.om.request.key.acl.OMKeyAddAclRequest;
import org.apache.hadoop.ozone.om.request.key.acl.OMKeyRemoveAclRequest;
@@ -129,6 +130,8 @@ public final class OzoneManagerRatisUtils {
return new OMKeysDeleteRequest(omRequest);
case RenameKey:
return new OMKeyRenameRequest(omRequest);
+ case RenameKeys:
+ return new OMKeysRenameRequest(omRequest);
case CreateDirectory:
return new OMDirectoryCreateRequest(omRequest);
case CreateFile:
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java
new file mode 100644
index 0000000..dbcde6d
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java
@@ -0,0 +1,271 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.request.key;
+
+import com.google.common.base.Optional;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
+import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
+import org.apache.hadoop.ozone.audit.AuditLogger;
+import org.apache.hadoop.ozone.audit.OMAction;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OMMetrics;
+import org.apache.hadoop.ozone.om.ResolvedBucket;
+import org.apache.hadoop.ozone.om.helpers.OmRenameKeys;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
+import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.om.response.key.OMKeysRenameResponse;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeysArgs;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeysMap;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeysRequest;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeysResponse;
+import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK;
+import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.PARTIAL_RENAME;
+import static org.apache.hadoop.ozone.OzoneConsts.RENAMED_KEYS_MAP;
+import static org.apache.hadoop.ozone.OzoneConsts.UNRENAMED_KEYS_MAP;
+import static
org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK;
+
+/**
+ * Handles rename keys request.
+ */
+public class OMKeysRenameRequest extends OMKeyRequest {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(OMKeysRenameRequest.class);
+
+ public OMKeysRenameRequest(OMRequest omRequest) {
+ super(omRequest);
+ }
+
+ @Override
+ @SuppressWarnings("methodlength")
+ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
+ long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) {
+
+ RenameKeysRequest renameKeysRequest =
getOmRequest().getRenameKeysRequest();
+ RenameKeysArgs renameKeysArgs = renameKeysRequest.getRenameKeysArgs();
+ String volumeName = renameKeysArgs.getVolumeName();
+ String bucketName = renameKeysArgs.getBucketName();
+ OMClientResponse omClientResponse = null;
+
+ List<RenameKeysMap> unRenamedKeys = new ArrayList<>();
+
+ // fromKeyName -> toKeyName
+ Map<String, String> renamedKeys = new HashMap<>();
+
+ Map<String, OmKeyInfo> fromKeyAndToKeyInfo = new HashMap<>();
+ OMMetrics omMetrics = ozoneManager.getMetrics();
+ omMetrics.incNumKeyRenames();
+
+ AuditLogger auditLogger = ozoneManager.getAuditLogger();
+
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
+
+ OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
+ IOException exception = null;
+ OmKeyInfo fromKeyValue = null;
+ Result result = null;
+ Map<String, String> auditMap = new LinkedHashMap<>();
+ String fromKeyName = null;
+ String toKeyName = null;
+ boolean acquiredLock = false;
+ boolean renameStatus = true;
+
+ try {
+ ResolvedBucket bucket = ozoneManager.resolveBucketLink(
+ Pair.of(volumeName, bucketName));
+ bucket.audit(auditMap);
+ volumeName = bucket.realVolume();
+ bucketName = bucket.realBucket();
+ acquiredLock =
+ omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK,
+ volumeName, bucketName);
+
+ for (RenameKeysMap renameKey : renameKeysArgs.getRenameKeysMapList()) {
+
+ fromKeyName = renameKey.getFromKeyName();
+ toKeyName = renameKey.getToKeyName();
+ RenameKeysMap.Builder unRenameKey = RenameKeysMap.newBuilder();
+
+ if (toKeyName.length() == 0 || fromKeyName.length() == 0) {
+ renameStatus = false;
+ unRenamedKeys.add(
+ unRenameKey.setFromKeyName(fromKeyName).setToKeyName(toKeyName)
+ .build());
+ LOG.error("Key name is empty fromKeyName {} toKeyName {}",
+ fromKeyName, toKeyName);
+ continue;
+ }
+
+ try {
+ // check Acls to see if user has access to perform delete operation
+ // on old key and create operation on new key
+ checkKeyAcls(ozoneManager, volumeName, bucketName, fromKeyName,
+ IAccessAuthorizer.ACLType.DELETE, OzoneObj.ResourceType.KEY);
+ checkKeyAcls(ozoneManager, volumeName, bucketName, toKeyName,
+ IAccessAuthorizer.ACLType.CREATE, OzoneObj.ResourceType.KEY);
+ } catch (Exception ex) {
+ renameStatus = false;
+ unRenamedKeys.add(
+ unRenameKey.setFromKeyName(fromKeyName).setToKeyName(toKeyName)
+ .build());
+ LOG.error("Acl check failed for fromKeyName {} toKeyName {}",
+ fromKeyName, toKeyName, ex);
+ continue;
+ }
+
+ // Check if toKey exists
+ String fromKey = omMetadataManager.getOzoneKey(volumeName, bucketName,
+ fromKeyName);
+ String toKey =
+ omMetadataManager.getOzoneKey(volumeName, bucketName, toKeyName);
+ OmKeyInfo toKeyValue = omMetadataManager.getKeyTable().get(toKey);
+
+ if (toKeyValue != null) {
+
+ renameStatus = false;
+ unRenamedKeys.add(
+ unRenameKey.setFromKeyName(fromKeyName).setToKeyName(toKeyName)
+ .build());
+ LOG.error("Received a request name of new key {} already exists",
+ toKeyName);
+ }
+
+ // fromKeyName should exist
+ fromKeyValue = omMetadataManager.getKeyTable().get(fromKey);
+ if (fromKeyValue == null) {
+ renameStatus = false;
+ unRenamedKeys.add(
+ unRenameKey.setFromKeyName(fromKeyName).setToKeyName(toKeyName)
+ .build());
+ LOG.error("Received a request to rename a Key does not exist {}",
+ fromKey);
+ continue;
+ }
+
+ fromKeyValue.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled());
+
+ fromKeyValue.setKeyName(toKeyName);
+
+ //Set modification time
+ fromKeyValue.setModificationTime(Time.now());
+
+ // Add to cache.
+ // fromKey should be deleted, toKey should be added with newly updated
+ // omKeyInfo.
+ Table<String, OmKeyInfo> keyTable = omMetadataManager.getKeyTable();
+ keyTable.addCacheEntry(new CacheKey<>(fromKey),
+ new CacheValue<>(Optional.absent(), trxnLogIndex));
+ keyTable.addCacheEntry(new CacheKey<>(toKey),
+ new CacheValue<>(Optional.of(fromKeyValue), trxnLogIndex));
+ renamedKeys.put(fromKeyName, toKeyName);
+ fromKeyAndToKeyInfo.put(fromKeyName, fromKeyValue);
+ }
+
+ OmRenameKeys newOmRenameKeys =
+ new OmRenameKeys(volumeName, bucketName, null, fromKeyAndToKeyInfo);
+ omClientResponse = new OMKeysRenameResponse(omResponse
+ .setRenameKeysResponse(RenameKeysResponse.newBuilder()
+ .setStatus(renameStatus)
+ .addAllUnRenamedKeys(unRenamedKeys))
+ .setStatus(renameStatus ? OK : PARTIAL_RENAME)
+ .setSuccess(renameStatus).build(),
+ newOmRenameKeys);
+
+ result = Result.SUCCESS;
+ } catch (IOException ex) {
+ result = Result.FAILURE;
+ exception = ex;
+ createErrorOMResponse(omResponse, ex);
+
+ omResponse.setRenameKeysResponse(RenameKeysResponse.newBuilder()
+ .setStatus(renameStatus).addAllUnRenamedKeys(unRenamedKeys).build());
+ omClientResponse = new OMKeysRenameResponse(omResponse.build());
+
+ } finally {
+ addResponseToDoubleBuffer(trxnLogIndex, omClientResponse,
+ omDoubleBufferHelper);
+ if (acquiredLock) {
+ omMetadataManager.getLock().releaseWriteLock(BUCKET_LOCK, volumeName,
+ bucketName);
+ }
+ }
+
+ auditMap = buildAuditMap(auditMap, renamedKeys, unRenamedKeys);
+ auditLog(auditLogger, buildAuditMessage(OMAction.RENAME_KEYS, auditMap,
+ exception, getOmRequest().getUserInfo()));
+
+ switch (result) {
+ case SUCCESS:
+ LOG.debug("Rename Keys is successfully completed for auditMap:{}.",
+ auditMap);
+ break;
+ case FAILURE:
+ ozoneManager.getMetrics().incNumKeyRenameFails();
+ LOG.error("Rename keys failed for auditMap:{}.", auditMap);
+ break;
+ default:
+ LOG.error("Unrecognized Result for OMKeysRenameRequest: {}",
+ renameKeysRequest);
+ }
+
+ return omClientResponse;
+ }
+
+ /**
+ * Build audit map for RenameKeys request.
+ *
+ * @param auditMap
+ * @param renamedKeys
+ * @param unRenameKeys
+ * @return
+ */
+ private Map<String, String> buildAuditMap(Map<String, String> auditMap,
+ Map<String, String> renamedKeys,
+ List<RenameKeysMap> unRenameKeys) {
+ Map<String, String> unRenameKeysMap = new HashMap<>();
+ for (RenameKeysMap renameKeysMap : unRenameKeys) {
+ unRenameKeysMap.put(renameKeysMap.getFromKeyName(),
+ renameKeysMap.getToKeyName());
+ }
+ auditMap.put(RENAMED_KEYS_MAP, renamedKeys.toString());
+ auditMap.put(UNRENAMED_KEYS_MAP, unRenameKeysMap.toString());
+ return auditMap;
+ }
+}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysRenameResponse.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysRenameResponse.java
new file mode 100644
index 0000000..a9ff7ad
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysRenameResponse.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.response.key;
+
+import org.apache.hadoop.hdds.utils.db.BatchOperation;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.helpers.OmRenameKeys;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
+import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+
+import javax.annotation.Nonnull;
+import java.io.IOException;
+import java.util.Map;
+
+import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.KEY_TABLE;
+
+/**
+ * Response for RenameKeys request.
+ */
+@CleanupTableInfo(cleanupTables = {KEY_TABLE})
+public class OMKeysRenameResponse extends OMClientResponse {
+
+ private OmRenameKeys omRenameKeys;
+
+ public OMKeysRenameResponse(@Nonnull OMResponse omResponse,
+ OmRenameKeys omRenameKeys) {
+ super(omResponse);
+ this.omRenameKeys = omRenameKeys;
+ }
+
+
+ /**
+ * For when the request is not successful or it is a replay transaction.
+ * For a successful request, the other constructor should be used.
+ */
+ public OMKeysRenameResponse(@Nonnull OMResponse omResponse) {
+ super(omResponse);
+ checkStatusNotOK();
+ }
+
+ @Override
+ public void addToDBBatch(OMMetadataManager omMetadataManager,
+ BatchOperation batchOperation) throws IOException {
+ String volumeName = omRenameKeys.getVolume();
+ String bucketName = omRenameKeys.getBucket();
+
+ for (Map.Entry< String, OmKeyInfo> entry :
+ omRenameKeys.getFromKeyAndToKeyInfo().entrySet()) {
+ String fromKeyName = entry.getKey();
+ OmKeyInfo newKeyInfo = entry.getValue();
+ String toKeyName = newKeyInfo.getKeyName();
+
+ omMetadataManager.getKeyTable().deleteWithBatch(batchOperation,
+ omMetadataManager
+ .getOzoneKey(volumeName, bucketName, fromKeyName));
+ omMetadataManager.getKeyTable().putWithBatch(batchOperation,
+ omMetadataManager.getOzoneKey(volumeName, bucketName, toKeyName),
+ newKeyInfo);
+ }
+ }
+
+}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeysRenameRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeysRenameRequest.java
new file mode 100644
index 0000000..9475906
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeysRenameRequest.java
@@ -0,0 +1,160 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.request.key;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.request.TestOMRequestUtils;
+import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeysArgs;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeysMap;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeysRequest;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+
+/**
+ * Tests RenameKey request.
+ */
+public class TestOMKeysRenameRequest extends TestOMKeyRequest {
+
+ private int count = 10;
+ private String parentDir = "/test";
+
+ @Test
+ public void testKeysRenameRequest() throws Exception {
+
+ OMRequest modifiedOmRequest = createRenameKeyRequest(false);
+
+ OMKeysRenameRequest omKeysRenameRequest =
+ new OMKeysRenameRequest(modifiedOmRequest);
+
+ OMClientResponse omKeysRenameResponse =
+ omKeysRenameRequest.validateAndUpdateCache(ozoneManager, 100L,
+ ozoneManagerDoubleBufferHelper);
+
+ Assert.assertTrue(omKeysRenameResponse.getOMResponse().getSuccess());
+ Assert.assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ omKeysRenameResponse.getOMResponse().getStatus());
+
+ for (int i = 0; i < count; i++) {
+ // Original key should be deleted, toKey should exist.
+ OmKeyInfo omKeyInfo = omMetadataManager.getKeyTable().get(
+ omMetadataManager.getOzoneKey(volumeName, bucketName,
+ parentDir.concat("/key" + i)));
+ Assert.assertNull(omKeyInfo);
+
+ omKeyInfo =
+ omMetadataManager.getKeyTable().get(omMetadataManager.getOzoneKey(
+ volumeName, bucketName, parentDir.concat("/newKey" + i)));
+ Assert.assertNotNull(omKeyInfo);
+ }
+
+ }
+
+ @Test
+ public void testKeysRenameRequestFail() throws Exception {
+ OMRequest modifiedOmRequest = createRenameKeyRequest(true);
+
+ OMKeysRenameRequest omKeysRenameRequest =
+ new OMKeysRenameRequest(modifiedOmRequest);
+
+ OMClientResponse omKeysRenameResponse =
+ omKeysRenameRequest.validateAndUpdateCache(ozoneManager, 100L,
+ ozoneManagerDoubleBufferHelper);
+
+ Assert.assertFalse(omKeysRenameResponse.getOMResponse().getSuccess());
+ Assert.assertEquals(OzoneManagerProtocolProtos.Status.PARTIAL_RENAME,
+ omKeysRenameResponse.getOMResponse().getStatus());
+
+ // The keys（key0 to key9）can be renamed success.
+ for (int i = 0; i < count; i++) {
+ // Original key should be deleted, toKey should exist.
+ OmKeyInfo omKeyInfo = omMetadataManager.getKeyTable().get(
+ omMetadataManager.getOzoneKey(volumeName, bucketName,
+ parentDir.concat("/key" + i)));
+ Assert.assertNull(omKeyInfo);
+
+ omKeyInfo =
+ omMetadataManager.getKeyTable().get(omMetadataManager.getOzoneKey(
+ volumeName, bucketName, parentDir.concat("/newKey" + i)));
+ Assert.assertNotNull(omKeyInfo);
+ }
+
+ // The key not rename should be in unRenamedKeys.
+ RenameKeysMap unRenamedKeys = omKeysRenameResponse.getOMResponse()
+ .getRenameKeysResponse().getUnRenamedKeys(0);
+ Assert.assertEquals("testKey", unRenamedKeys.getFromKeyName());
+ }
+
+ /**
+ * Create OMRequest which encapsulates RenameKeyRequest.
+ *
+ * @return OMRequest
+ */
+ private OMRequest createRenameKeyRequest(Boolean isIllegal) throws Exception
{
+
+ // Add volume, bucket and key entries to OM DB.
+ TestOMRequestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager);
+
+ List<RenameKeysMap> renameKeyList = new ArrayList<>();
+
+ for (int i = 0; i < count; i++) {
+ String key = parentDir.concat("/key" + i);
+ String toKey = parentDir.concat("/newKey" + i);
+ TestOMRequestUtils.addKeyToTableCache(volumeName, bucketName,
+ parentDir.concat("/key" + i), HddsProtos.ReplicationType.RATIS,
+ HddsProtos.ReplicationFactor.THREE, omMetadataManager);
+
+ RenameKeysMap.Builder renameKey = RenameKeysMap.newBuilder()
+ .setFromKeyName(key)
+ .setToKeyName(toKey);
+ renameKeyList.add(renameKey.build());
+ }
+
+
+ // Generating illegal data causes Rename Keys to fail.
+ if (isIllegal) {
+ RenameKeysMap.Builder renameKey = RenameKeysMap.newBuilder()
+ .setFromKeyName("testKey")
+ .setToKeyName("toKey");
+ renameKeyList.add(renameKey.build());
+ }
+
+ RenameKeysArgs.Builder renameKeyArgs = RenameKeysArgs.newBuilder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .addAllRenameKeysMap(renameKeyList);
+
+ RenameKeysRequest.Builder renameKeysReq = RenameKeysRequest.newBuilder()
+ .setRenameKeysArgs(renameKeyArgs.build());
+
+ return OMRequest.newBuilder()
+ .setClientId(UUID.randomUUID().toString())
+ .setRenameKeysRequest(renameKeysReq.build())
+ .setCmdType(OzoneManagerProtocolProtos.Type.RenameKeys).build();
+ }
+
+}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysRenameResponse.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysRenameResponse.java
new file mode 100644
index 0000000..a9db1b8
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysRenameResponse.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.response.key;
+
+import org.apache.hadoop.ozone.om.helpers.OmRenameKeys;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.request.TestOMRequestUtils;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeysResponse;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE;
+import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.RATIS;
+
+/**
+ * Tests OMKeyRenameResponse.
+ */
+public class TestOMKeysRenameResponse extends TestOMKeyResponse {
+ private OmRenameKeys omRenameKeys;
+ private int count = 10;
+ private String parentDir = "/test";
+
+ @Test
+ public void testKeysRenameResponse() throws Exception {
+
+ createPreRequisities();
+
+ OMResponse omResponse = OMResponse.newBuilder()
+ .setRenameKeysResponse(RenameKeysResponse.getDefaultInstance())
+ .setStatus(Status.OK).setCmdType(Type.RenameKeys).build();
+
+ OMKeysRenameResponse omKeysRenameResponse = new OMKeysRenameResponse(
+ omResponse, omRenameKeys);
+
+ omKeysRenameResponse.addToDBBatch(omMetadataManager, batchOperation);
+
+ // Do manual commit and see whether addToBatch is successful or not.
+ omMetadataManager.getStore().commitBatchOperation(batchOperation);
+
+ // Add volume, bucket and key entries to OM DB.
+ TestOMRequestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager);
+
+ for (int i = 0; i < count; i++) {
+ String key = parentDir.concat("/key" + i);
+ String toKey = parentDir.concat("/newKey" + i);
+ key = omMetadataManager.getOzoneKey(volumeName, bucketName, key);
+ toKey = omMetadataManager.getOzoneKey(volumeName, bucketName, toKey);
+ Assert.assertFalse(omMetadataManager.getKeyTable().isExist(key));
+ Assert.assertTrue(omMetadataManager.getKeyTable().isExist(toKey));
+ }
+ }
+
+ @Test
+ public void testKeysRenameResponseFail() throws Exception {
+
+ createPreRequisities();
+
+ OMResponse omResponse = OMResponse.newBuilder().setRenameKeysResponse(
+ RenameKeysResponse.getDefaultInstance())
+ .setStatus(Status.KEY_NOT_FOUND)
+ .setCmdType(Type.RenameKeys)
+ .build();
+
+ OMKeysRenameResponse omKeyRenameResponse = new OMKeysRenameResponse(
+ omResponse, omRenameKeys);
+
+ omKeyRenameResponse.checkAndUpdateDB(omMetadataManager, batchOperation);
+
+ // Do manual commit and see whether addToBatch is successful or not.
+ omMetadataManager.getStore().commitBatchOperation(batchOperation);
+
+ for (int i = 0; i < count; i++) {
+ String key = parentDir.concat("/key" + i);
+ String toKey = parentDir.concat("/newKey" + i);
+ key = omMetadataManager.getOzoneKey(volumeName, bucketName, key);
+ toKey = omMetadataManager.getOzoneKey(volumeName, bucketName, toKey);
+ // As omResponse has error, it is a no-op. So, no changes should happen.
+ Assert.assertTrue(omMetadataManager.getKeyTable().isExist(key));
+ Assert.assertFalse(omMetadataManager.getKeyTable().isExist(toKey));
+ }
+
+ }
+
+ private void createPreRequisities() throws Exception {
+
+ // Add volume, bucket and key entries to OM DB.
+ TestOMRequestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager);
+ Map<String, OmKeyInfo> formAndToKeyInfo = new HashMap<>();
+
+ for (int i = 0; i < count; i++) {
+ String key = parentDir.concat("/key" + i);
+ String toKey = parentDir.concat("/newKey" + i);
+ TestOMRequestUtils.addKeyToTable(false, volumeName,
+ bucketName, parentDir.concat("/key" + i), 0L, RATIS, THREE,
+ omMetadataManager);
+
+ OmKeyInfo omKeyInfo = omMetadataManager.getKeyTable().get(
+ omMetadataManager.getOzoneKey(volumeName, bucketName, key));
+ omKeyInfo.setKeyName(toKey);
+ formAndToKeyInfo.put(key, omKeyInfo);
+ }
+ omRenameKeys =
+ new OmRenameKeys(volumeName, bucketName, null, formAndToKeyInfo);
+
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
