This is an automated email from the ASF dual-hosted git repository.
bharat pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new c5aaad8 HDDS-3371. Cleanup of old write-path of key in OM (#2703)
c5aaad8 is described below
commit c5aaad8cdef758a0028e2be11ce0aea47b09011d
Author: GeorgeJahad <[email protected]>
AuthorDate: Mon Oct 25 20:23:09 2021 -0700
HDDS-3371. Cleanup of old write-path of key in OM (#2703)
---
.../ozone/om/protocol/OzoneManagerProtocol.java | 168 ++++-
.../dev-support/findbugsExcludeFile.xml | 4 +
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 24 +-
.../commandhandler/TestBlockDeletion.java | 10 +-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 22 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 273 ++++----
.../org/apache/hadoop/ozone/om/OzoneManager.java | 758 ---------------------
.../protocolPB/OzoneManagerRequestHandler.java | 25 -
8 files changed, 336 insertions(+), 948 deletions(-)
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
index 5b98a30..7bc67da 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -78,7 +78,11 @@ public interface OzoneManagerProtocol
* @param args - Arguments to create Volume.
* @throws IOException
*/
- void createVolume(OmVolumeArgs args) throws IOException;
+ default void createVolume(OmVolumeArgs args) throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented, as write requests use a new approach.");
+ }
+
/**
* Changes the owner of a volume.
@@ -88,7 +92,11 @@ public interface OzoneManagerProtocol
* already the owner.
* @throws IOException
*/
- boolean setOwner(String volume, String owner) throws IOException;
+ default boolean setOwner(String volume, String owner) throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented, as write requests use a new approach.");
+ }
+
/**
* Changes the Quota on a volume.
@@ -97,8 +105,12 @@ public interface OzoneManagerProtocol
* @param quotaInBytes - Volume quota in bytes.
* @throws IOException
*/
- void setQuota(String volume, long quotaInNamespace, long quotaInBytes)
- throws IOException;
+ default void setQuota(String volume, long quotaInNamespace, long
quotaInBytes)
+ throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented, as write requests use a new approach.");
+ }
+
/**
* Checks if the specified user can access this volume.
@@ -108,8 +120,11 @@ public interface OzoneManagerProtocol
* false otherwise
* @throws IOException
*/
- boolean checkVolumeAccess(String volume, OzoneAclInfo userAcl)
- throws IOException;
+ default boolean checkVolumeAccess(String volume, OzoneAclInfo userAcl)
+ throws IOException {
+ throw new UnsupportedOperationException("This operation is not
supported.");
+ }
+
/**
* Gets the volume information.
@@ -124,7 +139,11 @@ public interface OzoneManagerProtocol
* @param volume - Name of the volume.
* @throws IOException
*/
- void deleteVolume(String volume) throws IOException;
+ default void deleteVolume(String volume) throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented, as write requests use a new approach.");
+ }
+
/**
* Lists volumes accessible by a specific user.
@@ -154,7 +173,11 @@ public interface OzoneManagerProtocol
* @param bucketInfo - BucketInfo to create Bucket.
* @throws IOException
*/
- void createBucket(OmBucketInfo bucketInfo) throws IOException;
+ default void createBucket(OmBucketInfo bucketInfo) throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented, as write requests use a new approach.");
+ }
+
/**
* Gets the bucket information.
@@ -171,7 +194,11 @@ public interface OzoneManagerProtocol
* @param args - BucketArgs.
* @throws IOException
*/
- void setBucketProperty(OmBucketArgs args) throws IOException;
+ default void setBucketProperty(OmBucketArgs args) throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented, as write requests use a new approach.");
+ }
+
/**
* Open the given key and return an open key session.
@@ -180,7 +207,10 @@ public interface OzoneManagerProtocol
* @return OpenKeySession instance that client uses to talk to container.
* @throws IOException
*/
- OpenKeySession openKey(OmKeyArgs args) throws IOException;
+ default OpenKeySession openKey(OmKeyArgs args) throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented, as write requests use a new approach.");
+ }
/**
* Commit a key. This will make the change from the client visible. The
client
@@ -190,7 +220,12 @@ public interface OzoneManagerProtocol
* @param clientID the client identification
* @throws IOException
*/
- void commitKey(OmKeyArgs args, long clientID) throws IOException;
+ default void commitKey(OmKeyArgs args, long clientID)
+ throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented, as write requests use a new approach.");
+ }
+
/**
* Allocate a new block, it is assumed that the client is having an open key
@@ -203,9 +238,11 @@ public interface OzoneManagerProtocol
* @return an allocated block
* @throws IOException
*/
- OmKeyLocationInfo allocateBlock(OmKeyArgs args, long clientID,
- ExcludeList excludeList) throws IOException;
-
+ default OmKeyLocationInfo allocateBlock(OmKeyArgs args, long clientID,
+ ExcludeList excludeList) throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented, as write requests use a new approach.");
+ }
/**
* Look up for the container of an existing key.
@@ -222,7 +259,11 @@ public interface OzoneManagerProtocol
* @param toKeyName New name to be used for the Key
* @throws IOException
*/
- void renameKey(OmKeyArgs args, String toKeyName) throws IOException;
+ default void renameKey(OmKeyArgs args, String toKeyName) throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented, as write requests use a new approach.");
+ }
+
/**
* Rename existing keys within a bucket.
@@ -230,7 +271,11 @@ public interface OzoneManagerProtocol
* and fromKey name toKey info Map.
* @throws IOException
*/
- void renameKeys(OmRenameKeys omRenameKeys) throws IOException;
+ default void renameKeys(OmRenameKeys omRenameKeys) throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented, as write requests use a new approach.");
+ }
+
/**
* Deletes an existing key.
@@ -238,7 +283,11 @@ public interface OzoneManagerProtocol
* @param args the args of the key.
* @throws IOException
*/
- void deleteKey(OmKeyArgs args) throws IOException;
+ default void deleteKey(OmKeyArgs args) throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented, as write requests use a new approach.");
+ }
+
/**
* Deletes existing key/keys. This interface supports delete
@@ -248,7 +297,11 @@ public interface OzoneManagerProtocol
* @param deleteKeys
* @throws IOException
*/
- void deleteKeys(OmDeleteKeys deleteKeys) throws IOException;
+ default void deleteKeys(OmDeleteKeys deleteKeys) throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented, as write requests use a new approach.");
+ }
+
/**
* Deletes an existing empty bucket from volume.
@@ -256,7 +309,11 @@ public interface OzoneManagerProtocol
* @param bucket - Name of the bucket.
* @throws IOException
*/
- void deleteBucket(String volume, String bucket) throws IOException;
+ default void deleteBucket(String volume, String bucket) throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented, as write requests use a new approach.");
+ }
+
/**
* Returns a list of buckets represented by {@link OmBucketInfo}
@@ -389,7 +446,12 @@ public interface OzoneManagerProtocol
* @return MultipartInfo
* @throws IOException
*/
- OmMultipartInfo initiateMultipartUpload(OmKeyArgs keyArgs) throws
IOException;
+ default OmMultipartInfo initiateMultipartUpload(OmKeyArgs keyArgs)
+ throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented, as write requests use a new approach.");
+ }
+
/**
@@ -399,8 +461,12 @@ public interface OzoneManagerProtocol
* @return OmMultipartCommitUploadPartInfo
* @throws IOException
*/
- OmMultipartCommitUploadPartInfo commitMultipartUploadPart(
- OmKeyArgs omKeyArgs, long clientID) throws IOException;
+ default OmMultipartCommitUploadPartInfo commitMultipartUploadPart(
+ OmKeyArgs omKeyArgs, long clientID) throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented, as write requests use a new approach.");
+ }
+
/**
* Complete Multipart upload Request.
@@ -409,16 +475,24 @@ public interface OzoneManagerProtocol
* @return OmMultipartUploadCompleteInfo
* @throws IOException
*/
- OmMultipartUploadCompleteInfo completeMultipartUpload(
+ default OmMultipartUploadCompleteInfo completeMultipartUpload(
OmKeyArgs omKeyArgs, OmMultipartUploadCompleteList multipartUploadList)
- throws IOException;
+ throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented, as write requests use a new approach.");
+ }
+
/**
* Abort multipart upload.
* @param omKeyArgs
* @throws IOException
*/
- void abortMultipartUpload(OmKeyArgs omKeyArgs) throws IOException;
+ default void abortMultipartUpload(OmKeyArgs omKeyArgs) throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented, as write requests use a new approach.");
+ }
+
/**
* Returns list of parts of a multipart upload key.
@@ -446,14 +520,22 @@ public interface OzoneManagerProtocol
* @return S3SecretValue
* @throws IOException
*/
- S3SecretValue getS3Secret(String kerberosID) throws IOException;
+ default S3SecretValue getS3Secret(String kerberosID) throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented, as write requests use a new approach.");
+ }
+
/**
* Revokes s3Secret of given kerberos user.
* @param kerberosID
* @throws IOException
*/
- void revokeS3Secret(String kerberosID) throws IOException;
+ default void revokeS3Secret(String kerberosID) throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented, as write requests use a new approach.");
+ }
+
/**
* OzoneFS api to get file status for an entry.
@@ -476,7 +558,11 @@ public interface OzoneManagerProtocol
* @throws IOException if there is error in the db
* invalid arguments
*/
- void createDirectory(OmKeyArgs args) throws IOException;
+ default void createDirectory(OmKeyArgs args) throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented, as write requests use a new approach.");
+ }
+
/**
* OzoneFS api to creates an output stream for a file.
@@ -492,8 +578,12 @@ public interface OzoneManagerProtocol
* @throws IOException if there is error in the db
* invalid arguments
*/
- OpenKeySession createFile(OmKeyArgs keyArgs, boolean overWrite,
- boolean recursive) throws IOException;
+ default OpenKeySession createFile(OmKeyArgs keyArgs, boolean overWrite,
+ boolean recursive) throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented, as write requests use a new approach.");
+ }
+
/**
* OzoneFS api to lookup for a file.
@@ -528,7 +618,11 @@ public interface OzoneManagerProtocol
*
* @throws IOException if there is error.
* */
- boolean addAcl(OzoneObj obj, OzoneAcl acl) throws IOException;
+ default boolean addAcl(OzoneObj obj, OzoneAcl acl) throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented, as write requests use a new approach.");
+ }
+
/**
* Remove acl for Ozone object. Return true if acl is removed successfully
@@ -538,7 +632,11 @@ public interface OzoneManagerProtocol
*
* @throws IOException if there is error.
* */
- boolean removeAcl(OzoneObj obj, OzoneAcl acl) throws IOException;
+ default boolean removeAcl(OzoneObj obj, OzoneAcl acl) throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented, as write requests use a new approach.");
+ }
+
/**
* Acls to be set for given Ozone object. This operations reset ACL for
@@ -548,7 +646,11 @@ public interface OzoneManagerProtocol
*
* @throws IOException if there is error.
* */
- boolean setAcl(OzoneObj obj, List<OzoneAcl> acls) throws IOException;
+ default boolean setAcl(OzoneObj obj, List<OzoneAcl> acls) throws IOException
{
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented, as write requests use a new approach.");
+ }
+
/**
* Returns list of ACLs for given Ozone object.
diff --git a/hadoop-ozone/integration-test/dev-support/findbugsExcludeFile.xml
b/hadoop-ozone/integration-test/dev-support/findbugsExcludeFile.xml
index b089712..7078a67 100644
--- a/hadoop-ozone/integration-test/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-ozone/integration-test/dev-support/findbugsExcludeFile.xml
@@ -109,6 +109,10 @@
<Bug pattern="UWF_NULL_FIELD" />
</Match>
<Match>
+ <Class name="org.apache.hadoop.ozone.om.TestOmMetrics"/>
+ <Bug pattern="RV_RETURN_VALUE_IGNORED_NO_SIDE_EFFECT" />
+ </Match>
+ <Match>
<Class name="org.apache.hadoop.ozone.scm.TestContainerSmallFile"/>
<Bug pattern="DLS_DEAD_LOCAL_STORE" />
</Match>
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
index fd6eed5..403918b 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.hadoop.ozone.om.request.TestOMRequestUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.ozone.test.GenericTestUtils;
@@ -142,6 +143,7 @@ public class TestOzoneFileSystem {
private static boolean omRatisEnabled;
private static MiniOzoneCluster cluster;
+ private static OzoneManagerProtocol writeClient;
private static FileSystem fs;
private static OzoneFileSystem o3fs;
private static String volumeName;
@@ -168,6 +170,8 @@ public class TestOzoneFileSystem {
.build();
cluster.waitForClusterToBeReady();
+ writeClient = cluster.getRpcClient().getObjectStore()
+ .getClientProxy().getOzoneManagerClient();
// create a volume and a bucket to be used by OzoneFileSystem
OzoneBucket bucket =
TestDataUtil.createVolumeAndBucket(cluster, bucketLayout);
@@ -580,15 +584,29 @@ public class TestOzoneFileSystem {
.setLocationInfoList(new ArrayList<>())
.build();
- OpenKeySession session = cluster.getOzoneManager().openKey(keyArgs);
- cluster.getOzoneManager().commitKey(keyArgs, session.getId());
+ OpenKeySession session = writeClient.openKey(keyArgs);
+ writeClient.commitKey(keyArgs, session.getId());
Path parent = new Path("/");
+
+ // Wait until the filestatus is updated
+ if (!enabledFileSystemPaths) {
+ GenericTestUtils.waitFor(()-> {
+ try {
+ return fs.listStatus(parent).length!=0;
+ } catch (IOException e) {
+ LOG.error("listStatus() Failed", e);
+ Assert.fail("listStatus() Failed");
+ return false;
+ }
+ }, 1000, 120000);
+ }
+
FileStatus[] fileStatuses = fs.listStatus(parent);
// the number of immediate children of root is 1
Assert.assertEquals(1, fileStatuses.length);
- cluster.getOzoneManager().deleteKey(keyArgs);
+ writeClient.deleteKey(keyArgs);
}
/**
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
index 4f1d229..8977f9e 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.ozone.test.GenericTestUtils;
import org.apache.ozone.test.GenericTestUtils.LogCapturer;
import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB;
@@ -103,6 +104,7 @@ public class TestBlockDeletion {
private MiniOzoneCluster cluster = null;
private StorageContainerManager scm = null;
private OzoneManager om = null;
+ private OzoneManagerProtocol writeClient;
private Set<Long> containerIdsWithDeletedBlocks;
private long maxTransactionId = 0;
private File baseDir;
@@ -156,6 +158,8 @@ public class TestBlockDeletion {
cluster.waitForClusterToBeReady();
store = OzoneClientFactory.getRpcClient(conf).getObjectStore();
om = cluster.getOzoneManager();
+ writeClient = cluster.getRpcClient().getObjectStore()
+ .getClientProxy().getOzoneManagerClient();
scm = cluster.getStorageContainerManager();
containerIdsWithDeletedBlocks = new HashSet<>();
metrics = scm.getScmBlockManager().getSCMBlockDeletingService()
@@ -218,7 +222,7 @@ public class TestBlockDeletion {
matchContainerTransactionIds();
Assert.assertEquals(0L, metrics.getNumBlockDeletionTransactionCreated());
- om.deleteKey(keyArgs);
+ writeClient.deleteKey(keyArgs);
Thread.sleep(5000);
// The blocks should not be deleted in the DN as the container is open
try {
@@ -334,7 +338,7 @@ public class TestBlockDeletion {
.setState(ContainerProtos.ContainerDataProto.State.CLOSED));
});
- om.deleteKey(keyArgs);
+ writeClient.deleteKey(keyArgs);
// Wait for blocks to be deleted and container reports to be processed
Thread.sleep(5000);
containerInfos = scm.getContainerManager().getContainers();
@@ -505,7 +509,7 @@ public class TestBlockDeletion {
new RatisReplicationConfig(HddsProtos.ReplicationFactor.THREE))
.setRefreshPipeline(true)
.build();
- om.deleteKey(keyArgs);
+ writeClient.deleteKey(keyArgs);
}
// Wait for block delete command sent from OM
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmBlockVersioning.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmBlockVersioning.java
index 02adb4d..a57ba1c 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmBlockVersioning.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmBlockVersioning.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
+import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.commons.lang3.RandomStringUtils;
import org.junit.AfterClass;
@@ -57,6 +58,7 @@ public class TestOmBlockVersioning {
private static MiniOzoneCluster cluster = null;
private static OzoneConfiguration conf;
private static OzoneManager ozoneManager;
+ private static OzoneManagerProtocol writeClient;
@Rule
public ExpectedException exception = ExpectedException.none();
@@ -74,6 +76,8 @@ public class TestOmBlockVersioning {
cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady();
ozoneManager = cluster.getOzoneManager();
+ writeClient = cluster.getRpcClient().getObjectStore()
+ .getClientProxy().getOzoneManagerClient();
}
/**
@@ -108,11 +112,11 @@ public class TestOmBlockVersioning {
.build();
// 1st update, version 0
- OpenKeySession openKey = ozoneManager.openKey(keyArgs);
+ OpenKeySession openKey = writeClient.openKey(keyArgs);
// explicitly set the keyLocation list before committing the key.
keyArgs.setLocationInfoList(openKey.getKeyInfo().getLatestVersionLocations()
.getBlocksLatestVersionOnly());
- ozoneManager.commitKey(keyArgs, openKey.getId());
+ writeClient.commitKey(keyArgs, openKey.getId());
OmKeyInfo keyInfo = ozoneManager.lookupKey(keyArgs);
OmKeyLocationInfoGroup highestVersion =
@@ -121,13 +125,13 @@ public class TestOmBlockVersioning {
assertEquals(1, highestVersion.getLocationList().size());
// 2nd update, version 1
- openKey = ozoneManager.openKey(keyArgs);
+ openKey = writeClient.openKey(keyArgs);
//OmKeyLocationInfo locationInfo =
- // ozoneManager.allocateBlock(keyArgs, openKey.getId());
+ // writeClient.allocateBlock(keyArgs, openKey.getId());
// explicitly set the keyLocation list before committing the key.
keyArgs.setLocationInfoList(openKey.getKeyInfo().getLatestVersionLocations()
.getBlocksLatestVersionOnly());
- ozoneManager.commitKey(keyArgs, openKey.getId());
+ writeClient.commitKey(keyArgs, openKey.getId());
keyInfo = ozoneManager.lookupKey(keyArgs);
highestVersion = checkVersions(keyInfo.getKeyLocationVersions());
@@ -135,11 +139,11 @@ public class TestOmBlockVersioning {
assertEquals(1, highestVersion.getLocationList().size());
// 3rd update, version 2
- openKey = ozoneManager.openKey(keyArgs);
+ openKey = writeClient.openKey(keyArgs);
// this block will be appended to the latest version of version 2.
OmKeyLocationInfo locationInfo =
- ozoneManager.allocateBlock(keyArgs, openKey.getId(),
+ writeClient.allocateBlock(keyArgs, openKey.getId(),
new ExcludeList());
List<OmKeyLocationInfo> locationInfoList =
openKey.getKeyInfo().getLatestVersionLocations()
@@ -147,7 +151,7 @@ public class TestOmBlockVersioning {
Assert.assertTrue(locationInfoList.size() == 1);
locationInfoList.add(locationInfo);
keyArgs.setLocationInfoList(locationInfoList);
- ozoneManager.commitKey(keyArgs, openKey.getId());
+ writeClient.commitKey(keyArgs, openKey.getId());
keyInfo = ozoneManager.lookupKey(keyArgs);
highestVersion = checkVersions(keyInfo.getKeyLocationVersions());
@@ -214,4 +218,4 @@ public class TestOmBlockVersioning {
assertEquals(1,
keyInfo.getLatestVersionLocations().getLocationList().size());
}
-}
\ No newline at end of file
+}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
index a5410d8..9410973 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
@@ -22,30 +22,39 @@ import static
org.apache.hadoop.ozone.security.acl.OzoneObj.ResourceType.VOLUME;
import static org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType.OZONE;
import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyInt;
-import static org.mockito.Matchers.anyLong;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
+import java.util.UUID;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.client.ContainerBlockID;
import org.apache.hadoop.hdds.scm.HddsWhiteboxTestUtils;
+import org.apache.hadoop.hdds.scm.pipeline.MockPipeline;
+import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneAcl;
+import org.apache.hadoop.ozone.TestDataUtil;
import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
+import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
+import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
+import org.assertj.core.util.Lists;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -68,11 +77,12 @@ public class TestOmMetrics {
private MiniOzoneCluster.Builder clusterBuilder;
private OzoneConfiguration conf;
private OzoneManager ozoneManager;
-
+ private OzoneManagerProtocol writeClient;
/**
* The exception used for testing failure metrics.
*/
- private IOException exception = new IOException();
+ private final OMException exception =
+ new OMException("dummyException", OMException.ResultCodes.TIMEOUT);
/**
* Create a MiniDFSCluster for testing.
@@ -89,6 +99,8 @@ public class TestOmMetrics {
cluster = clusterBuilder.build();
cluster.waitForClusterToBeReady();
ozoneManager = cluster.getOzoneManager();
+ writeClient = cluster.getRpcClient().getObjectStore()
+ .getClientProxy().getOzoneManagerClient();
}
/**
@@ -111,31 +123,24 @@ public class TestOmMetrics {
ozoneManager, "volumeManager");
VolumeManager mockVm = Mockito.spy(volumeManager);
- Mockito.doNothing().when(mockVm).createVolume(null);
- Mockito.doNothing().when(mockVm).deleteVolume(null);
- Mockito.doReturn(null).when(mockVm).getVolumeInfo(null);
- Mockito.doReturn(true).when(mockVm).checkVolumeAccess(null, null);
- Mockito.doNothing().when(mockVm).setOwner(null, null);
- Mockito.doReturn(null).when(mockVm).listVolumes(null, null, null, 0);
-
- HddsWhiteboxTestUtils.setInternalState(
- ozoneManager, "volumeManager", mockVm);
- doVolumeOps();
-
+ OmVolumeArgs volumeArgs = createVolumeArgs();
+ doVolumeOps(volumeArgs);
MetricsRecordBuilder omMetrics = getMetrics("OMMetrics");
- assertCounter("NumVolumeOps", 6L, omMetrics);
+ assertCounter("NumVolumeOps", 5L, omMetrics);
assertCounter("NumVolumeCreates", 1L, omMetrics);
assertCounter("NumVolumeUpdates", 1L, omMetrics);
assertCounter("NumVolumeInfos", 1L, omMetrics);
- assertCounter("NumVolumeCheckAccesses", 1L, omMetrics);
assertCounter("NumVolumeDeletes", 1L, omMetrics);
assertCounter("NumVolumeLists", 1L, omMetrics);
assertCounter("NumVolumes", 1L, omMetrics);
- ozoneManager.createVolume(null);
- ozoneManager.createVolume(null);
- ozoneManager.createVolume(null);
- ozoneManager.deleteVolume(null);
+ volumeArgs = createVolumeArgs();
+ writeClient.createVolume(volumeArgs);
+ volumeArgs = createVolumeArgs();
+ writeClient.createVolume(volumeArgs);
+ volumeArgs = createVolumeArgs();
+ writeClient.createVolume(volumeArgs);
+ writeClient.deleteVolume(volumeArgs.getVolume());
omMetrics = getMetrics("OMMetrics");
@@ -143,31 +148,29 @@ public class TestOmMetrics {
assertCounter("NumVolumes", 3L, omMetrics);
- // inject exception to test for Failure Metrics
- Mockito.doThrow(exception).when(mockVm).createVolume(null);
- Mockito.doThrow(exception).when(mockVm).deleteVolume(null);
- Mockito.doThrow(exception).when(mockVm).getVolumeInfo(null);
- Mockito.doThrow(exception).when(mockVm).checkVolumeAccess(null, null);
- Mockito.doThrow(exception).when(mockVm).setOwner(null, null);
- Mockito.doThrow(exception).when(mockVm).listVolumes(null, null, null, 0);
+ // inject exception to test for Failure Metrics on the read path
+ Mockito.doThrow(exception).when(mockVm).getVolumeInfo(any());
+ Mockito.doThrow(exception).when(mockVm).listVolumes(any(), any(),
+ any(), anyInt());
HddsWhiteboxTestUtils.setInternalState(ozoneManager,
"volumeManager", mockVm);
- doVolumeOps();
+ // inject exception to test for Failure Metrics on the write path
+ mockWritePathExceptions(OmVolumeArgs.class);
+ volumeArgs = createVolumeArgs();
+ doVolumeOps(volumeArgs);
omMetrics = getMetrics("OMMetrics");
- assertCounter("NumVolumeOps", 16L, omMetrics);
+ assertCounter("NumVolumeOps", 14L, omMetrics);
assertCounter("NumVolumeCreates", 5L, omMetrics);
assertCounter("NumVolumeUpdates", 2L, omMetrics);
assertCounter("NumVolumeInfos", 2L, omMetrics);
- assertCounter("NumVolumeCheckAccesses", 2L, omMetrics);
assertCounter("NumVolumeDeletes", 3L, omMetrics);
assertCounter("NumVolumeLists", 2L, omMetrics);
assertCounter("NumVolumeCreateFails", 1L, omMetrics);
assertCounter("NumVolumeUpdateFails", 1L, omMetrics);
assertCounter("NumVolumeInfoFails", 1L, omMetrics);
- assertCounter("NumVolumeCheckAccessFails", 1L, omMetrics);
assertCounter("NumVolumeDeleteFails", 1L, omMetrics);
assertCounter("NumVolumeListFails", 1L, omMetrics);
@@ -189,16 +192,8 @@ public class TestOmMetrics {
ozoneManager, "bucketManager");
BucketManager mockBm = Mockito.spy(bucketManager);
- Mockito.doNothing().when(mockBm).createBucket(null);
- Mockito.doNothing().when(mockBm).deleteBucket(null, null);
- Mockito.doReturn(null).when(mockBm).getBucketInfo(null, null);
- Mockito.doNothing().when(mockBm).setBucketProperty(null);
- Mockito.doReturn(null).when(mockBm).listBuckets(null, null, null, 0);
-
- HddsWhiteboxTestUtils.setInternalState(
- ozoneManager, "bucketManager", mockBm);
-
- doBucketOps();
+ OmBucketInfo bucketInfo = createBucketInfo();
+ doBucketOps(bucketInfo);
MetricsRecordBuilder omMetrics = getMetrics("OMMetrics");
assertCounter("NumBucketOps", 5L, omMetrics);
@@ -209,24 +204,29 @@ public class TestOmMetrics {
assertCounter("NumBucketLists", 1L, omMetrics);
assertCounter("NumBuckets", 0L, omMetrics);
- ozoneManager.createBucket(null);
- ozoneManager.createBucket(null);
- ozoneManager.createBucket(null);
- ozoneManager.deleteBucket(null, null);
+ bucketInfo = createBucketInfo();
+ writeClient.createBucket(bucketInfo);
+ bucketInfo = createBucketInfo();
+ writeClient.createBucket(bucketInfo);
+ bucketInfo = createBucketInfo();
+ writeClient.createBucket(bucketInfo);
+ writeClient.deleteBucket(bucketInfo.getVolumeName(),
+ bucketInfo.getBucketName());
omMetrics = getMetrics("OMMetrics");
assertCounter("NumBuckets", 2L, omMetrics);
- // inject exception to test for Failure Metrics
- Mockito.doThrow(exception).when(mockBm).createBucket(null);
- Mockito.doThrow(exception).when(mockBm).deleteBucket(null, null);
- Mockito.doThrow(exception).when(mockBm).getBucketInfo(null, null);
- Mockito.doThrow(exception).when(mockBm).setBucketProperty(null);
- Mockito.doThrow(exception).when(mockBm).listBuckets(null, null, null, 0);
+ // inject exception to test for Failure Metrics on the read path
+ Mockito.doThrow(exception).when(mockBm).getBucketInfo(any(), any());
+ Mockito.doThrow(exception).when(mockBm).listBuckets(any(), any(),
+ any(), anyInt());
HddsWhiteboxTestUtils.setInternalState(
ozoneManager, "bucketManager", mockBm);
- doBucketOps();
+
+ // inject exception to test for Failure Metrics on the write path
+ mockWritePathExceptions(OmBucketInfo.class);
+ doBucketOps(bucketInfo);
omMetrics = getMetrics("OMMetrics");
assertCounter("NumBucketOps", 14L, omMetrics);
@@ -250,31 +250,17 @@ public class TestOmMetrics {
@Test
public void testKeyOps() throws Exception {
+ // This test needs a cluster with DNs and SCM to wait on safemode
+ clusterBuilder.setNumDatanodes(3);
+ conf.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_ENABLED, true);
startCluster();
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
KeyManager keyManager = (KeyManager) HddsWhiteboxTestUtils
.getInternalState(ozoneManager, "keyManager");
KeyManager mockKm = Mockito.spy(keyManager);
- BucketManager mockBm = Mockito.mock(BucketManager.class);
-
- OmBucketInfo mockBucket = OmBucketInfo.newBuilder()
- .setVolumeName("").setBucketName("")
- .build();
- Mockito.when(mockBm.getBucketInfo(any(), any())).thenReturn(mockBucket);
- Mockito.doReturn(null).when(mockKm).openKey(any());
- Mockito.doNothing().when(mockKm).deleteKey(any());
- Mockito.doReturn(null).when(mockKm).lookupKey(any(), any());
- Mockito.doReturn(null).when(mockKm).listKeys(any(), any(), any(), any(),
- anyInt());
- Mockito.doReturn(null).when(mockKm).listTrash(any(), any(), any(), any(),
- anyInt());
- Mockito.doNothing().when(mockKm).commitKey(any(), anyLong());
- Mockito.doReturn(null).when(mockKm).initiateMultipartUpload(any());
-
- HddsWhiteboxTestUtils.setInternalState(
- ozoneManager, "bucketManager", mockBm);
- HddsWhiteboxTestUtils.setInternalState(
- ozoneManager, "keyManager", mockKm);
- OmKeyArgs keyArgs = createKeyArgs();
+ TestDataUtil.createVolumeAndBucket(cluster, volumeName, bucketName);
+ OmKeyArgs keyArgs = createKeyArgs(volumeName, bucketName);
doKeyOps(keyArgs);
MetricsRecordBuilder omMetrics = getMetrics("OMMetrics");
@@ -287,32 +273,33 @@ public class TestOmMetrics {
assertCounter("NumKeys", 0L, omMetrics);
assertCounter("NumInitiateMultipartUploads", 1L, omMetrics);
-
- ozoneManager.openKey(keyArgs);
- ozoneManager.commitKey(keyArgs, 0);
- ozoneManager.openKey(keyArgs);
- ozoneManager.commitKey(keyArgs, 0);
- ozoneManager.openKey(keyArgs);
- ozoneManager.commitKey(keyArgs, 0);
- ozoneManager.deleteKey(keyArgs);
+ keyArgs = createKeyArgs(volumeName, bucketName);
+ OpenKeySession keySession = writeClient.openKey(keyArgs);
+ writeClient.commitKey(keyArgs, keySession.getId());
+ keyArgs = createKeyArgs(volumeName, bucketName);
+ keySession = writeClient.openKey(keyArgs);
+ writeClient.commitKey(keyArgs, keySession.getId());
+ keyArgs = createKeyArgs(volumeName, bucketName);
+ keySession = writeClient.openKey(keyArgs);
+ writeClient.commitKey(keyArgs, keySession.getId());
+ writeClient.deleteKey(keyArgs);
omMetrics = getMetrics("OMMetrics");
assertCounter("NumKeys", 2L, omMetrics);
- // inject exception to test for Failure Metrics
- Mockito.doThrow(exception).when(mockKm).openKey(any());
- Mockito.doThrow(exception).when(mockKm).deleteKey(any());
+ // inject exception to test for Failure Metrics on the read path
Mockito.doThrow(exception).when(mockKm).lookupKey(any(), any());
Mockito.doThrow(exception).when(mockKm).listKeys(
any(), any(), any(), any(), anyInt());
Mockito.doThrow(exception).when(mockKm).listTrash(
any(), any(), any(), any(), anyInt());
- Mockito.doThrow(exception).when(mockKm).commitKey(any(), anyLong());
- Mockito.doThrow(exception).when(mockKm).initiateMultipartUpload(any());
-
HddsWhiteboxTestUtils.setInternalState(
ozoneManager, "keyManager", mockKm);
+
+ // inject exception to test for Failure Metrics on the write path
+ mockWritePathExceptions(OmBucketInfo.class);
+ keyArgs = createKeyArgs(volumeName, bucketName);
doKeyOps(keyArgs);
omMetrics = getMetrics("OMMetrics");
@@ -339,6 +326,30 @@ public class TestOmMetrics {
}
+ private <T> void mockWritePathExceptions(Class<T>klass) throws Exception {
+ String tableName;
+ if (klass == OmBucketInfo.class) {
+ tableName = "bucketTable";
+ } else {
+ tableName = "volumeTable";
+ }
+ OMMetadataManager metadataManager = (OMMetadataManager)
+ HddsWhiteboxTestUtils.getInternalState(ozoneManager,
"metadataManager");
+ OMMetadataManager mockMm = Mockito.spy(metadataManager);
+ @SuppressWarnings("unchecked")
+ Table<String, T> table = (Table<String, T>)
+ HddsWhiteboxTestUtils.getInternalState(metadataManager, tableName);
+ Table<String, T> mockTable = Mockito.spy(table);
+ Mockito.doThrow(exception).when(mockTable).isExist(any());
+ if (klass == OmBucketInfo.class) {
+ Mockito.doReturn(mockTable).when(mockMm).getBucketTable();
+ } else {
+ Mockito.doReturn(mockTable).when(mockMm).getVolumeTable();
+ }
+ HddsWhiteboxTestUtils.setInternalState(
+ ozoneManager, "metadataManager", mockMm);
+ }
+
@Test
public void testAclOperations() throws Exception {
startCluster();
@@ -355,19 +366,19 @@ public class TestOmMetrics {
assertCounter("NumGetAcl", 1L, omMetrics);
// Test addAcl
- ozoneManager.addAcl(volObj,
+ writeClient.addAcl(volObj,
new OzoneAcl(IAccessAuthorizer.ACLIdentityType.USER, "ozoneuser",
IAccessAuthorizer.ACLType.ALL, ACCESS));
omMetrics = getMetrics("OMMetrics");
assertCounter("NumAddAcl", 1L, omMetrics);
// Test setAcl
- ozoneManager.setAcl(volObj, acls);
+ writeClient.setAcl(volObj, acls);
omMetrics = getMetrics("OMMetrics");
assertCounter("NumSetAcl", 1L, omMetrics);
// Test removeAcl
- ozoneManager.removeAcl(volObj, acls.get(0));
+ writeClient.removeAcl(volObj, acls.get(0));
omMetrics = getMetrics("OMMetrics");
assertCounter("NumRemoveAcl", 1L, omMetrics);
@@ -442,34 +453,29 @@ public class TestOmMetrics {
/**
* Test volume operations with ignoring thrown exception.
*/
- private void doVolumeOps() {
+ private void doVolumeOps(OmVolumeArgs volumeArgs) {
try {
- ozoneManager.createVolume(null);
+ writeClient.createVolume(volumeArgs);
} catch (IOException ignored) {
}
try {
- ozoneManager.deleteVolume(null);
+ ozoneManager.getVolumeInfo(volumeArgs.getVolume());
} catch (IOException ignored) {
}
try {
- ozoneManager.getVolumeInfo(null);
+ writeClient.setOwner(volumeArgs.getVolume(), "dummy");
} catch (IOException ignored) {
}
try {
- ozoneManager.checkVolumeAccess(null, null);
+ ozoneManager.listAllVolumes("", null, 0);
} catch (IOException ignored) {
}
try {
- ozoneManager.setOwner(null, null);
- } catch (IOException ignored) {
- }
-
- try {
- ozoneManager.listAllVolumes(null, null, 0);
+ writeClient.deleteVolume(volumeArgs.getVolume());
} catch (IOException ignored) {
}
}
@@ -477,29 +483,29 @@ public class TestOmMetrics {
/**
* Test bucket operations with ignoring thrown exception.
*/
- private void doBucketOps() {
+ private void doBucketOps(OmBucketInfo info) {
try {
- ozoneManager.createBucket(null);
+ writeClient.createBucket(info);
} catch (IOException ignored) {
}
try {
- ozoneManager.deleteBucket(null, null);
+ ozoneManager.getBucketInfo(info.getVolumeName(), info.getBucketName());
} catch (IOException ignored) {
}
try {
- ozoneManager.getBucketInfo(null, null);
+ writeClient.setBucketProperty(getBucketArgs(info));
} catch (IOException ignored) {
}
try {
- ozoneManager.setBucketProperty(null);
+ ozoneManager.listBuckets(info.getVolumeName(), null, null, 0);
} catch (IOException ignored) {
}
try {
- ozoneManager.listBuckets(null, null, null, 0);
+ writeClient.deleteBucket(info.getVolumeName(), info.getBucketName());
} catch (IOException ignored) {
}
}
@@ -508,13 +514,15 @@ public class TestOmMetrics {
* Test key operations with ignoring thrown exception.
*/
private void doKeyOps(OmKeyArgs keyArgs) {
+ OpenKeySession keySession = null;
try {
- ozoneManager.openKey(keyArgs);
+ keySession = writeClient.openKey(keyArgs);
} catch (IOException ignored) {
}
try {
- ozoneManager.deleteKey(keyArgs);
+ long id = (keySession != null)?keySession.getId():0;
+ writeClient.commitKey(keyArgs, id);
} catch (IOException ignored) {
}
@@ -524,35 +532,66 @@ public class TestOmMetrics {
}
try {
- ozoneManager.listKeys("", "", null, null, 0);
+ ozoneManager.listKeys(keyArgs.getVolumeName(),
+ keyArgs.getBucketName(), null, null, 0);
} catch (IOException ignored) {
}
try {
- ozoneManager.listTrash("", "", null, null, 0);
+ ozoneManager.listTrash(keyArgs.getVolumeName(),
+ keyArgs.getBucketName(), null, null, 0);
} catch (IOException ignored) {
}
try {
- ozoneManager.commitKey(keyArgs, 0);
+ writeClient.deleteKey(keyArgs);
} catch (IOException ignored) {
}
try {
- ozoneManager.initiateMultipartUpload(keyArgs);
+ writeClient.initiateMultipartUpload(keyArgs);
} catch (IOException ignored) {
}
-
}
- private OmKeyArgs createKeyArgs() {
+ private OmKeyArgs createKeyArgs(String volumeName, String bucketName)
+ throws IOException {
OmKeyLocationInfo keyLocationInfo = new OmKeyLocationInfo.Builder()
.setBlockID(new BlockID(new ContainerBlockID(1, 1)))
+ .setPipeline(MockPipeline.createSingleNodePipeline())
.build();
keyLocationInfo.setCreateVersion(0);
+ String keyName = UUID.randomUUID().toString();
return new OmKeyArgs.Builder()
.setLocationInfoList(Collections.singletonList(keyLocationInfo))
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(keyName)
+ .setAcls(Lists.emptyList())
+ .build();
+ }
+ private OmVolumeArgs createVolumeArgs() {
+ String volumeName = UUID.randomUUID().toString();
+ return new OmVolumeArgs.Builder()
+ .setVolume(volumeName)
+ .setOwnerName("dummy")
+ .setAdminName("dummyAdmin")
+ .build();
+ }
+ private OmBucketArgs getBucketArgs(OmBucketInfo info) {
+ return new OmBucketArgs.Builder()
+ .setVolumeName(info.getVolumeName())
+ .setBucketName(info.getBucketName())
+ .build();
+ }
+ private OmBucketInfo createBucketInfo() throws IOException {
+ OmVolumeArgs volumeArgs = createVolumeArgs();
+ writeClient.createVolume(volumeArgs);
+ String bucketName = UUID.randomUUID().toString();
+ return new OmBucketInfo.Builder()
+ .setVolumeName(volumeArgs.getVolume())
+ .setBucketName(bucketName)
.build();
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index cb554f5..783cf84 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -67,7 +67,6 @@ import
org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCer
import
org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
-import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.hdds.scm.ha.SCMNodeInfo;
import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
@@ -116,24 +115,14 @@ import
org.apache.hadoop.ozone.om.exceptions.OMNotLeaderException;
import org.apache.hadoop.ozone.om.ha.OMHANodeDetails;
import org.apache.hadoop.ozone.om.helpers.OMNodeDetails;
import org.apache.hadoop.ozone.om.helpers.DBUpdates;
-import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
-import org.apache.hadoop.ozone.om.helpers.OmDeleteKeys;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
-import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
-import org.apache.hadoop.ozone.om.helpers.OmMultipartCommitUploadPartInfo;
-import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
-import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
-import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteList;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts;
-import org.apache.hadoop.ozone.om.helpers.OmRenameKeys;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
-import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
-import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
import org.apache.hadoop.ozone.om.protocol.OMInterServiceProtocol;
@@ -154,7 +143,6 @@ import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DBUpdatesRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleInfo;
-import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort;
import org.apache.hadoop.ozone.protocolPB.OMInterServiceProtocolServerSideImpl;
import
org.apache.hadoop.ozone.storage.proto.OzoneManagerStorageProtos.PersistedUserVolumeInfo;
@@ -251,7 +239,6 @@ import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOU
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.DETECTED_LOOP_IN_BUCKET_LINKS;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_AUTH_METHOD;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST;
-import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TOKEN_ERROR_OTHER;
import static
org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.VOLUME_LOCK;
import static
org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer.RaftServerStatus.LEADER_AND_READY;
@@ -1963,34 +1950,6 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
}
/**
- * Creates a volume.
- *
- * @param args - Arguments to create Volume.
- * @throws IOException
- */
- @Override
- public void createVolume(OmVolumeArgs args) throws IOException {
- try {
- metrics.incNumVolumeCreates();
- if (isAclEnabled) {
- checkAcls(ResourceType.VOLUME, StoreType.OZONE, ACLType.CREATE,
- args.getVolume(), null, null);
- }
- volumeManager.createVolume(args);
- AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.CREATE_VOLUME,
- (args == null) ? null : args.toAuditMap()));
- metrics.incNumVolumes();
- } catch (Exception ex) {
- metrics.incNumVolumeCreateFails();
- AUDIT.logWriteFailure(
- buildAuditMessageForFailure(OMAction.CREATE_VOLUME,
- (args == null) ? null : args.toAuditMap(), ex)
- );
- throw ex;
- }
- }
-
- /**
* Checks if current caller has acl permissions.
*
* @param resType - Type of ozone resource. Ex volume, bucket.
@@ -2151,84 +2110,6 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
}
/**
- * {@inheritDoc}
- */
- @Override
- public boolean setOwner(String volume, String owner) throws IOException {
- if (isAclEnabled) {
- checkAcls(ResourceType.VOLUME, StoreType.OZONE, ACLType.WRITE_ACL,
volume,
- null, null);
- }
- Map<String, String> auditMap = buildAuditMap(volume);
- auditMap.put(OzoneConsts.OWNER, owner);
- try {
- metrics.incNumVolumeUpdates();
- volumeManager.setOwner(volume, owner);
- AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.SET_OWNER,
- auditMap));
- return true;
- } catch (Exception ex) {
- metrics.incNumVolumeUpdateFails();
- AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.SET_OWNER,
- auditMap, ex)
- );
- throw ex;
- }
- }
-
- /**
- * Changes the Quota on a volume.
- *
- * @param volume - Name of the volume.
- * @param quotaInNamespace - Volume quota in counts.
- * @param quotaInBytes - Volume quota in bytes.
- * @throws IOException
- */
- @Override
- public void setQuota(String volume, long quotaInNamespace,
- long quotaInBytes) throws IOException {
- throw new UnsupportedOperationException("OzoneManager does not require " +
- "this to be implemented. As this requests use a new approach");
- }
-
- /**
- * Checks if the specified user can access this volume.
- *
- * @param volume - volume
- * @param userAcl - user acls which needs to be checked for access
- * @return true if the user has required access for the volume, false
- * otherwise
- * @throws IOException
- */
- @Override
- public boolean checkVolumeAccess(String volume, OzoneAclInfo userAcl)
- throws IOException {
- if (isAclEnabled) {
- checkAcls(ResourceType.VOLUME, StoreType.OZONE,
- ACLType.READ, volume, null, null);
- }
- boolean auditSuccess = true;
- Map<String, String> auditMap = buildAuditMap(volume);
- auditMap.put(OzoneConsts.USER_ACL,
- (userAcl == null) ? null : userAcl.getName());
- try {
- metrics.incNumVolumeCheckAccesses();
- return volumeManager.checkVolumeAccess(volume, userAcl);
- } catch (Exception ex) {
- metrics.incNumVolumeCheckAccessFails();
- auditSuccess = false;
- AUDIT.logReadFailure(buildAuditMessageForFailure(
- OMAction.CHECK_VOLUME_ACCESS, auditMap, ex));
- throw ex;
- } finally {
- if (auditSuccess) {
- AUDIT.logReadSuccess(buildAuditMessageForSuccess(
- OMAction.CHECK_VOLUME_ACCESS, auditMap));
- }
- }
- }
-
- /**
* Gets the volume information.
*
* @param volume - Volume name.
@@ -2262,32 +2143,6 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
}
/**
- * Deletes an existing empty volume.
- *
- * @param volume - Name of the volume.
- * @throws IOException
- */
- @Override
- public void deleteVolume(String volume) throws IOException {
- try {
- if (isAclEnabled) {
- checkAcls(ResourceType.VOLUME, StoreType.OZONE, ACLType.DELETE, volume,
- null, null);
- }
- metrics.incNumVolumeDeletes();
- volumeManager.deleteVolume(volume);
- AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.DELETE_VOLUME,
- buildAuditMap(volume)));
- metrics.decNumVolumes();
- } catch (Exception ex) {
- metrics.incNumVolumeDeleteFails();
- AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.DELETE_VOLUME,
- buildAuditMap(volume), ex));
- throw ex;
- }
- }
-
- /**
* Lists volumes accessible by a specific user.
*
* @param userName - user name
@@ -2393,32 +2248,6 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
}
/**
- * Creates a bucket.
- *
- * @param bucketInfo - BucketInfo to create bucket.
- * @throws IOException
- */
- @Override
- public void createBucket(OmBucketInfo bucketInfo) throws IOException {
- try {
- if (isAclEnabled) {
- checkAcls(ResourceType.VOLUME, StoreType.OZONE, ACLType.CREATE,
- bucketInfo.getVolumeName(), bucketInfo.getBucketName(), null);
- }
- metrics.incNumBucketCreates();
- bucketManager.createBucket(bucketInfo);
- AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.CREATE_BUCKET,
- (bucketInfo == null) ? null : bucketInfo.toAuditMap()));
- metrics.incNumBuckets();
- } catch (Exception ex) {
- metrics.incNumBucketCreateFails();
- AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.CREATE_BUCKET,
- (bucketInfo == null) ? null : bucketInfo.toAuditMap(), ex));
- throw ex;
- }
- }
-
- /**
* {@inheritDoc}
*/
@Override
@@ -2489,149 +2318,6 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
}
/**
- * Allocate a key.
- *
- * @param args - attributes of the key.
- * @return OpenKeySession - a handler to key that client uses to
- * talk to container.
- * @throws IOException
- */
- @Override
- public OpenKeySession openKey(OmKeyArgs args) throws IOException {
- ResolvedBucket bucket = resolveBucketLink(args);
-
- if (isAclEnabled) {
- try {
- checkAcls(ResourceType.KEY, StoreType.OZONE, ACLType.WRITE,
- bucket.realVolume(), bucket.realBucket(), args.getKeyName());
- } catch (OMException ex) {
- // For new keys key checkAccess call will fail as key doesn't exist.
- // Check user access for bucket.
- if (ex.getResult().equals(KEY_NOT_FOUND)) {
- checkAcls(ResourceType.BUCKET, StoreType.OZONE, ACLType.WRITE,
- bucket.realVolume(), bucket.realBucket(), args.getKeyName());
- } else {
- throw ex;
- }
- }
- }
-
- boolean auditSuccess = true;
- Map<String, String> auditMap = bucket.audit(args.toAuditMap());
-
- args = bucket.update(args);
-
- try {
- metrics.incNumKeyAllocates();
- return keyManager.openKey(args);
- } catch (Exception ex) {
- metrics.incNumKeyAllocateFails();
- auditSuccess = false;
- AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.ALLOCATE_KEY,
- auditMap, ex));
- throw ex;
- } finally {
- if (auditSuccess) {
- AUDIT.logWriteSuccess(buildAuditMessageForSuccess(
- OMAction.ALLOCATE_KEY, auditMap));
- }
- }
- }
-
- @Override
- public void commitKey(OmKeyArgs args, long clientID)
- throws IOException {
- ResolvedBucket bucket = resolveBucketLink(args);
-
- if (isAclEnabled) {
- try {
- checkAcls(ResourceType.KEY, StoreType.OZONE, ACLType.WRITE,
- bucket.realVolume(), bucket.realBucket(), args.getKeyName());
- } catch (OMException ex) {
- // For new keys key checkAccess call will fail as key doesn't exist.
- // Check user access for bucket.
- if (ex.getResult().equals(KEY_NOT_FOUND)) {
- checkAcls(ResourceType.BUCKET, StoreType.OZONE, ACLType.WRITE,
- bucket.realVolume(), bucket.realBucket(), args.getKeyName());
- } else {
- throw ex;
- }
- }
- }
-
- Map<String, String> auditMap = bucket.audit(args.toAuditMap());
- auditMap.put(OzoneConsts.CLIENT_ID, String.valueOf(clientID));
-
- args = bucket.update(args);
-
- try {
- metrics.incNumKeyCommits();
- keyManager.commitKey(args, clientID);
- AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.COMMIT_KEY,
- auditMap));
- // As when we commit the key it is visible, so we should increment here.
- // As key also can have multiple versions, we need to increment keys
- // only if version is 0. Currently we have not complete support of
- // versioning of keys. So, this can be revisited later.
- if (args.getLocationInfoList() != null &&
- args.getLocationInfoList().size() > 0 &&
- args.getLocationInfoList().get(0) != null &&
- args.getLocationInfoList().get(0).getCreateVersion() == 0) {
- metrics.incNumKeys();
- }
- } catch (Exception ex) {
- metrics.incNumKeyCommitFails();
- AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.COMMIT_KEY,
- auditMap, ex));
- throw ex;
- }
- }
-
- @Override
- public OmKeyLocationInfo allocateBlock(OmKeyArgs args, long clientID,
- ExcludeList excludeList) throws IOException {
- ResolvedBucket bucket = resolveBucketLink(args);
-
- if (isAclEnabled) {
- try {
- checkAcls(ResourceType.KEY, StoreType.OZONE, ACLType.WRITE,
- bucket.realVolume(), bucket.realBucket(), args.getKeyName());
- } catch (OMException ex) {
- // For new keys key checkAccess call will fail as key doesn't exist.
- // Check user access for bucket.
- if (ex.getResult().equals(KEY_NOT_FOUND)) {
- checkAcls(ResourceType.BUCKET, StoreType.OZONE, ACLType.WRITE,
- bucket.realVolume(), bucket.realBucket(), args.getKeyName());
- } else {
- throw ex;
- }
- }
- }
-
- boolean auditSuccess = true;
- Map<String, String> auditMap = bucket.audit(args.toAuditMap());
- auditMap.put(OzoneConsts.CLIENT_ID, String.valueOf(clientID));
-
- args = bucket.update(args);
-
- try {
- metrics.incNumBlockAllocateCalls();
- return keyManager.allocateBlock(args, clientID, excludeList);
- } catch (Exception ex) {
- metrics.incNumBlockAllocateCallFails();
- auditSuccess = false;
-
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.ALLOCATE_BLOCK,
- auditMap, ex));
- throw ex;
- } finally {
- if (auditSuccess) {
- AUDIT.logWriteSuccess(buildAuditMessageForSuccess(
- OMAction.ALLOCATE_BLOCK, auditMap));
- }
- }
- }
-
- /**
* Lookup a key.
*
* @param args - attributes of the key.
@@ -2669,86 +2355,6 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
}
}
-
- @Override
- public void renameKeys(OmRenameKeys omRenameKeys)
- throws IOException {
- throw new UnsupportedOperationException("OzoneManager does not require " +
- "this to be implemented. As write requests use a new approach");
- }
-
- @Override
- public void renameKey(OmKeyArgs args, String toKeyName) throws IOException {
- Preconditions.checkNotNull(args);
-
- ResolvedBucket bucket = resolveBucketLink(args);
-
- if (isAclEnabled) {
- checkAcls(ResourceType.KEY, StoreType.OZONE, ACLType.WRITE,
- bucket.realVolume(), bucket.realBucket(), args.getKeyName());
- }
-
- Map<String, String> auditMap = bucket.audit(args.toAuditMap());
- auditMap.put(OzoneConsts.TO_KEY_NAME, toKeyName);
-
- args = bucket.update(args);
-
- try {
- metrics.incNumKeyRenames();
- keyManager.renameKey(args, toKeyName);
- AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.RENAME_KEY,
- auditMap));
- } catch (IOException e) {
- metrics.incNumKeyRenameFails();
- AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.RENAME_KEY,
- auditMap, e));
- throw e;
- }
- }
-
- /**
- * Deletes an existing key.
- *
- * @param args - attributes of the key.
- * @throws IOException
- */
- @Override
- public void deleteKey(OmKeyArgs args) throws IOException {
- Map<String, String> auditMap = args.toAuditMap();
- try {
- ResolvedBucket bucket = resolveBucketLink(args);
- args = bucket.update(args);
-
- if (isAclEnabled) {
- checkAcls(ResourceType.KEY, StoreType.OZONE, ACLType.DELETE,
- bucket.realVolume(), bucket.realBucket(), args.getKeyName());
- }
-
- metrics.incNumKeyDeletes();
- keyManager.deleteKey(args);
- AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.DELETE_KEY,
- auditMap));
- metrics.decNumKeys();
- } catch (Exception ex) {
- metrics.incNumKeyDeleteFails();
- AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.DELETE_KEY,
- auditMap, ex));
- throw ex;
- }
- }
-
- /**
- * Deletes an existing key.
- *
- * @param deleteKeys - List of keys to be deleted from volume and a bucket.
- * @throws IOException
- */
- @Override
- public void deleteKeys(OmDeleteKeys deleteKeys) throws IOException {
- throw new UnsupportedOperationException("OzoneManager does not require " +
- "this to be implemented. As write requests use a new approach");
- }
-
@Override
public List<OmKeyInfo> listKeys(String volumeName, String bucketName,
String startKey, String keyPrefix, int maxKeys) throws IOException {
@@ -2821,61 +2427,6 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
}
}
- /**
- * Sets bucket property from args.
- *
- * @param args - BucketArgs.
- * @throws IOException
- */
- @Override
- public void setBucketProperty(OmBucketArgs args)
- throws IOException {
- if (isAclEnabled) {
- checkAcls(ResourceType.BUCKET, StoreType.OZONE, ACLType.WRITE,
- args.getVolumeName(), args.getBucketName(), null);
- }
- try {
- metrics.incNumBucketUpdates();
- bucketManager.setBucketProperty(args);
- AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.UPDATE_BUCKET,
- (args == null) ? null : args.toAuditMap()));
- } catch (Exception ex) {
- metrics.incNumBucketUpdateFails();
- AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.UPDATE_BUCKET,
- (args == null) ? null : args.toAuditMap(), ex));
- throw ex;
- }
- }
-
- /**
- * Deletes an existing empty bucket from volume.
- *
- * @param volume - Name of the volume.
- * @param bucket - Name of the bucket.
- * @throws IOException
- */
- @Override
- public void deleteBucket(String volume, String bucket) throws IOException {
- if (isAclEnabled) {
- checkAcls(ResourceType.BUCKET, StoreType.OZONE, ACLType.WRITE, volume,
- bucket, null);
- }
- Map<String, String> auditMap = buildAuditMap(volume);
- auditMap.put(OzoneConsts.BUCKET, bucket);
- try {
- metrics.incNumBucketDeletes();
- bucketManager.deleteBucket(volume, bucket);
- AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.DELETE_BUCKET,
- auditMap));
- metrics.decNumBuckets();
- } catch (Exception ex) {
- metrics.incNumBucketDeleteFails();
- AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.DELETE_BUCKET,
- auditMap, ex));
- throw ex;
- }
- }
-
private Map<String, String> buildAuditMap(String volume) {
Map<String, String> auditMap = new LinkedHashMap<>();
auditMap.put(OzoneConsts.VOLUME, volume);
@@ -3056,135 +2607,6 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
}
@Override
- /**
- * {@inheritDoc}
- */
- public S3SecretValue getS3Secret(String kerberosID) throws IOException {
- UserGroupInformation user = ProtobufRpcEngine.Server.getRemoteUser();
-
- // Check whether user name passed is matching with the current user or not.
- if (!user.getUserName().equals(kerberosID)) {
- throw new OMException("User mismatch. Requested user name is " +
- "mismatched " + kerberosID + ", with current user " +
- user.getUserName(), OMException.ResultCodes.USER_MISMATCH);
- }
- return s3SecretManager.getS3Secret(kerberosID);
- }
-
- @Override
- /**
- * {@inheritDoc}
- */
- public void revokeS3Secret(String kerberosID) {
- throw new UnsupportedOperationException("OzoneManager does not require " +
- "this to be implemented. As write requests use a new approach");
- }
-
- @Override
- public OmMultipartInfo initiateMultipartUpload(OmKeyArgs keyArgs) throws
- IOException {
-
- Preconditions.checkNotNull(keyArgs);
- ResolvedBucket bucket = resolveBucketLink(keyArgs);
-
- Map<String, String> auditMap = bucket.audit(keyArgs.toAuditMap());
-
- keyArgs = bucket.update(keyArgs);
-
- metrics.incNumInitiateMultipartUploads();
- try {
- OmMultipartInfo result = keyManager.initiateMultipartUpload(keyArgs);
- AUDIT.logWriteSuccess(buildAuditMessageForSuccess(
- OMAction.INITIATE_MULTIPART_UPLOAD, auditMap));
- return result;
- } catch (IOException ex) {
- AUDIT.logWriteFailure(buildAuditMessageForFailure(
- OMAction.INITIATE_MULTIPART_UPLOAD, auditMap, ex));
- metrics.incNumInitiateMultipartUploadFails();
- throw ex;
- }
- }
-
- @Override
- public OmMultipartCommitUploadPartInfo commitMultipartUploadPart(
- OmKeyArgs keyArgs, long clientID) throws IOException {
-
- Preconditions.checkNotNull(keyArgs);
- ResolvedBucket bucket = resolveBucketLink(keyArgs);
-
- Map<String, String> auditMap = bucket.audit(keyArgs.toAuditMap());
-
- keyArgs = bucket.update(keyArgs);
-
- metrics.incNumCommitMultipartUploadParts();
- try {
- OmMultipartCommitUploadPartInfo result =
- keyManager.commitMultipartUploadPart(keyArgs, clientID);
- AUDIT.logWriteSuccess(buildAuditMessageForSuccess(
- OMAction.COMMIT_MULTIPART_UPLOAD_PARTKEY, auditMap));
- return result;
- } catch (IOException ex) {
- AUDIT.logWriteFailure(buildAuditMessageForFailure(
- OMAction.INITIATE_MULTIPART_UPLOAD, auditMap, ex));
- metrics.incNumCommitMultipartUploadPartFails();
- throw ex;
- }
- }
-
- @Override
- public OmMultipartUploadCompleteInfo completeMultipartUpload(
- OmKeyArgs omKeyArgs, OmMultipartUploadCompleteList multipartUploadList)
- throws IOException {
-
- Preconditions.checkNotNull(omKeyArgs);
- ResolvedBucket bucket = resolveBucketLink(omKeyArgs);
-
- Map<String, String> auditMap = bucket.audit(omKeyArgs.toAuditMap());
- auditMap.put(OzoneConsts.MULTIPART_LIST, multipartUploadList
- .getMultipartMap().toString());
-
- omKeyArgs = bucket.update(omKeyArgs);
-
- metrics.incNumCompleteMultipartUploads();
- try {
- OmMultipartUploadCompleteInfo result =
keyManager.completeMultipartUpload(
- omKeyArgs, multipartUploadList);
- AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction
- .COMPLETE_MULTIPART_UPLOAD, auditMap));
- return result;
- } catch (IOException ex) {
- metrics.incNumCompleteMultipartUploadFails();
- AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction
- .COMPLETE_MULTIPART_UPLOAD, auditMap, ex));
- throw ex;
- }
- }
-
- @Override
- public void abortMultipartUpload(OmKeyArgs omKeyArgs) throws IOException {
-
- Preconditions.checkNotNull(omKeyArgs);
- ResolvedBucket bucket = resolveBucketLink(omKeyArgs);
-
- Map<String, String> auditMap = bucket.audit(omKeyArgs.toAuditMap());
-
- omKeyArgs = bucket.update(omKeyArgs);
-
- metrics.incNumAbortMultipartUploads();
- try {
- keyManager.abortMultipartUpload(omKeyArgs);
- AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction
- .COMPLETE_MULTIPART_UPLOAD, auditMap));
- } catch (IOException ex) {
- metrics.incNumAbortMultipartUploadFails();
- AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction
- .COMPLETE_MULTIPART_UPLOAD, auditMap, ex));
- throw ex;
- }
-
- }
-
- @Override
public OmMultipartUploadListParts listParts(final String volumeName,
final String bucketName, String keyName, String uploadID,
int partNumberMarker, int maxParts) throws IOException {
@@ -3275,59 +2697,6 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
}
@Override
- public void createDirectory(OmKeyArgs args) throws IOException {
- ResolvedBucket bucket = resolveBucketLink(args);
-
- boolean auditSuccess = true;
- Map<String, String> auditMap = bucket.audit(args.toAuditMap());
-
- args = bucket.update(args);
-
- try {
- metrics.incNumCreateDirectory();
- keyManager.createDirectory(args);
- } catch (IOException ex) {
- metrics.incNumCreateDirectoryFails();
- auditSuccess = false;
- AUDIT.logWriteFailure(
- buildAuditMessageForFailure(OMAction.CREATE_DIRECTORY, auditMap,
ex));
- throw ex;
- } finally {
- if (auditSuccess) {
- AUDIT.logWriteSuccess(
- buildAuditMessageForSuccess(OMAction.CREATE_DIRECTORY, auditMap));
- }
- }
- }
-
- @Override
- public OpenKeySession createFile(OmKeyArgs args, boolean overWrite,
- boolean recursive) throws IOException {
- ResolvedBucket bucket = resolveBucketLink(args);
-
- boolean auditSuccess = true;
- Map<String, String> auditMap = bucket.audit(args.toAuditMap());
-
- args = bucket.update(args);
-
- try {
- metrics.incNumCreateFile();
- return keyManager.createFile(args, overWrite, recursive);
- } catch (Exception ex) {
- metrics.incNumCreateFileFails();
- auditSuccess = false;
- AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.CREATE_FILE,
- auditMap, ex));
- throw ex;
- } finally {
- if (auditSuccess) {
- AUDIT.logWriteSuccess(buildAuditMessageForSuccess(
- OMAction.CREATE_FILE, auditMap));
- }
- }
- }
-
- @Override
public OmKeyInfo lookupFile(OmKeyArgs args) throws IOException {
ResolvedBucket bucket = resolveBucketLink(args);
@@ -3409,133 +2778,6 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
}
/**
- * Add acl for Ozone object. Return true if acl is added successfully else
- * false.
- *
- * @param obj Ozone object for which acl should be added.
- * @param acl ozone acl to be added.
- * @throws IOException if there is error.
- */
- @Override
- public boolean addAcl(OzoneObj obj, OzoneAcl acl) throws IOException {
- boolean auditSuccess = true;
-
- try {
- if (isAclEnabled) {
- checkAcls(obj.getResourceType(), obj.getStoreType(), ACLType.WRITE_ACL,
- obj.getVolumeName(), obj.getBucketName(), obj.getKeyName());
- }
- metrics.incNumAddAcl();
- switch (obj.getResourceType()) {
- case VOLUME:
- return volumeManager.addAcl(obj, acl);
- case BUCKET:
- return bucketManager.addAcl(obj, acl);
- case KEY:
- return keyManager.addAcl(obj, acl);
- case PREFIX:
- return prefixManager.addAcl(obj, acl);
- default:
- throw new OMException("Unexpected resource type: " +
- obj.getResourceType(), INVALID_REQUEST);
- }
- } catch (Exception ex) {
- auditSuccess = false;
- auditAcl(obj, Arrays.asList(acl), OMAction.ADD_ACL, ex);
- throw ex;
- } finally {
- if (auditSuccess) {
- auditAcl(obj, Arrays.asList(acl), OMAction.ADD_ACL, null);
- }
- }
- }
-
- /**
- * Remove acl for Ozone object. Return true if acl is removed successfully
- * else false.
- *
- * @param obj Ozone object.
- * @param acl Ozone acl to be removed.
- * @throws IOException if there is error.
- */
- @Override
- public boolean removeAcl(OzoneObj obj, OzoneAcl acl) throws IOException {
- boolean auditSuccess = true;
-
- try {
- if (isAclEnabled) {
- checkAcls(obj.getResourceType(), obj.getStoreType(), ACLType.WRITE_ACL,
- obj.getVolumeName(), obj.getBucketName(), obj.getKeyName());
- }
- metrics.incNumRemoveAcl();
- switch (obj.getResourceType()) {
- case VOLUME:
- return volumeManager.removeAcl(obj, acl);
- case BUCKET:
- return bucketManager.removeAcl(obj, acl);
- case KEY:
- return keyManager.removeAcl(obj, acl);
- case PREFIX:
- return prefixManager.removeAcl(obj, acl);
-
- default:
- throw new OMException("Unexpected resource type: " +
- obj.getResourceType(), INVALID_REQUEST);
- }
- } catch (Exception ex) {
- auditSuccess = false;
- auditAcl(obj, Arrays.asList(acl), OMAction.REMOVE_ACL, ex);
- throw ex;
- } finally {
- if (auditSuccess) {
- auditAcl(obj, Arrays.asList(acl), OMAction.REMOVE_ACL, null);
- }
- }
- }
-
- /**
- * Acls to be set for given Ozone object. This operations reset ACL for given
- * object to list of ACLs provided in argument.
- *
- * @param obj Ozone object.
- * @param acls List of acls.
- * @throws IOException if there is error.
- */
- @Override
- public boolean setAcl(OzoneObj obj, List<OzoneAcl> acls) throws IOException {
- boolean auditSuccess = true;
-
- try {
- if (isAclEnabled) {
- checkAcls(obj.getResourceType(), obj.getStoreType(), ACLType.WRITE_ACL,
- obj.getVolumeName(), obj.getBucketName(), obj.getKeyName());
- }
- metrics.incNumSetAcl();
- switch (obj.getResourceType()) {
- case VOLUME:
- return volumeManager.setAcl(obj, acls);
- case BUCKET:
- return bucketManager.setAcl(obj, acls);
- case KEY:
- return keyManager.setAcl(obj, acls);
- case PREFIX:
- return prefixManager.setAcl(obj, acls);
- default:
- throw new OMException("Unexpected resource type: " +
- obj.getResourceType(), INVALID_REQUEST);
- }
- } catch (Exception ex) {
- auditSuccess = false;
- auditAcl(obj, acls, OMAction.SET_ACL, ex);
- throw ex;
- } finally {
- if (auditSuccess) {
- auditAcl(obj, acls, OMAction.SET_ACL, null);
- }
- }
- }
-
- /**
* Returns list of ACLs for given Ozone object.
*
* @param obj Ozone object.
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index 61587a3..6459e43 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@ -24,7 +24,6 @@ import java.util.stream.Collectors;
import
org.apache.hadoop.hdds.protocol.proto.HddsProtos.UpgradeFinalizationStatus;
import org.apache.hadoop.hdds.client.ReplicationConfig;
-import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
import org.apache.hadoop.hdds.utils.db.SequenceNumberNotFoundException;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.om.OzoneManager;
@@ -34,7 +33,6 @@ import org.apache.hadoop.ozone.om.helpers.DBUpdates;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
-import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts;
import org.apache.hadoop.ozone.om.helpers.OmPartInfo;
@@ -49,8 +47,6 @@ import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
-import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AllocateBlockRequest;
-import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AllocateBlockResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CheckVolumeAccessRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CheckVolumeAccessResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.FinalizeUpgradeProgressRequest;
@@ -455,27 +451,6 @@ public class OzoneManagerRequestHandler implements
RequestHandler {
return resp.build();
}
- private AllocateBlockResponse allocateBlock(AllocateBlockRequest request,
- int clientVersion) throws IOException {
- AllocateBlockResponse.Builder resp =
- AllocateBlockResponse.newBuilder();
-
- KeyArgs keyArgs = request.getKeyArgs();
- OmKeyArgs omKeyArgs = new OmKeyArgs.Builder()
- .setVolumeName(keyArgs.getVolumeName())
- .setBucketName(keyArgs.getBucketName())
- .setKeyName(keyArgs.getKeyName())
- .build();
-
- OmKeyLocationInfo newLocation = impl.allocateBlock(omKeyArgs,
- request.getClientID(), ExcludeList.getFromProtoBuf(
- request.getExcludeList()));
-
- resp.setKeyLocation(newLocation.getProtobuf(clientVersion));
-
- return resp.build();
- }
-
private ServiceListResponse getServiceList(ServiceListRequest request)
throws IOException {
ServiceListResponse.Builder resp = ServiceListResponse.newBuilder();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
