This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new d08a4c1 HDDS-3728. Bucket space: check quotaUsageInBytes when write
key and allocate block. (#1458)
d08a4c1 is described below
commit d08a4c1d973f7752875436f497c4a007bea9f250
Author: micah zhao <[email protected]>
AuthorDate: Fri Oct 9 19:42:53 2020 +0800
HDDS-3728. Bucket space: check quotaUsageInBytes when write key and
allocate block. (#1458)
---
.../client/rpc/TestOzoneRpcClientAbstract.java | 67 ++++++++++++++++++++--
.../ozone/om/request/file/OMFileCreateRequest.java | 3 +-
.../om/request/key/OMAllocateBlockRequest.java | 3 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 3 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 21 +++++++
5 files changed, 88 insertions(+), 9 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
index c9551fd..b7b75a4 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
@@ -815,6 +815,7 @@ public abstract class TestOzoneRpcClientAbstract {
}
@Test
+ @SuppressWarnings("methodlength")
public void testCheckUsedBytesQuota() throws IOException {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
@@ -828,13 +829,15 @@ public abstract class TestOzoneRpcClientAbstract {
store.createVolume(volumeName);
volume = store.getVolume(volumeName);
+
+ // Test volume quota.
// Set quota In Bytes for a smaller value
store.getVolume(volumeName).setQuota(
OzoneQuota.parseQuota("1 Bytes", 100));
volume.createBucket(bucketName);
OzoneBucket bucket = volume.getBucket(bucketName);
- // Test write key.
+ // Test volume quota: write key.
// The remaining quota does not satisfy a block size, so the write fails.
try {
writeKey(bucket, UUID.randomUUID().toString(), ONE, value, valueLength);
@@ -845,7 +848,7 @@ public abstract class TestOzoneRpcClientAbstract {
// Write failed, volume usedBytes should be 0
Assert.assertEquals(0L, store.getVolume(volumeName).getUsedBytes());
- // Test write file.
+ // Test volume quota: write file.
// The remaining quota does not satisfy a block size, so the write fails.
try {
writeFile(bucket, UUID.randomUUID().toString(), ONE, value, 0);
@@ -856,7 +859,7 @@ public abstract class TestOzoneRpcClientAbstract {
// Write failed, volume usedBytes should be 0
Assert.assertEquals(0L, store.getVolume(volumeName).getUsedBytes());
- // Write a key(with two blocks), test allocateBlock fails.
+ // Test volume quota: write key(with two blocks), test allocateBlock fails.
store.getVolume(volumeName).setQuota(
OzoneQuota.parseQuota(blockSize + "Bytes", 100));
try {
@@ -873,8 +876,8 @@ public abstract class TestOzoneRpcClientAbstract {
// AllocateBlock failed, volume usedBytes should be 1 * blockSize.
Assert.assertEquals(blockSize, store.getVolume(volumeName).getUsedBytes());
- // Write large key(with five blocks), the first four blocks will succeed，
- // while the later block will fail.
+ // Test volume quota: write large key(with five blocks), the first four
+ // blocks will succeed，while the later block will fail.
store.getVolume(volumeName).setQuota(
OzoneQuota.parseQuota(5 * blockSize + "Bytes", 100));
try {
@@ -892,7 +895,59 @@ public abstract class TestOzoneRpcClientAbstract {
Assert.assertEquals(5 * blockSize,
store.getVolume(volumeName).getUsedBytes());
- Assert.assertEquals(4, countException);
+ // Test bucket quota.
+ // Set quota In Bytes for a smaller value
+ store.getVolume(volumeName).setQuota(
+ OzoneQuota.parseQuota(Long.MAX_VALUE + " Bytes", 100));
+ bucketName = UUID.randomUUID().toString();
+ volume.createBucket(bucketName);
+ bucket = volume.getBucket(bucketName);
+ bucket.setQuota(OzoneQuota.parseQuota("1 Bytes", 100));
+
+ // Test bucket quota: write key.
+ // The remaining quota does not satisfy a block size, so the write fails.
+ try {
+ writeKey(bucket, UUID.randomUUID().toString(), ONE, value, valueLength);
+ } catch (IOException ex) {
+ countException++;
+ GenericTestUtils.assertExceptionContains("QUOTA_EXCEEDED", ex);
+ }
+ // Write failed, bucket usedBytes should be 0
+ Assert.assertEquals(0L,
+ store.getVolume(volumeName).getBucket(bucketName).getUsedBytes());
+
+ // Test bucket quota: write file.
+ // The remaining quota does not satisfy a block size, so the write fails.
+ try {
+ writeFile(bucket, UUID.randomUUID().toString(), ONE, value, 0);
+ } catch (IOException ex) {
+ countException++;
+ GenericTestUtils.assertExceptionContains("QUOTA_EXCEEDED", ex);
+ }
+ // Write failed, bucket usedBytes should be 0
+ Assert.assertEquals(0L,
+ store.getVolume(volumeName).getBucket(bucketName).getUsedBytes());
+
+ // Test bucket quota: write large key(with five blocks), the first four
+ // blocks will succeed，while the later block will fail.
+ bucket.setQuota(OzoneQuota.parseQuota(
+ 4 * blockSize + " Bytes", 100));
+ try {
+ OzoneOutputStream out = bucket.createKey(UUID.randomUUID().toString(),
+ valueLength, STAND_ALONE, ONE, new HashMap<>());
+ for (int i = 0; i <= (4 * blockSize) / value.length(); i++) {
+ out.write(value.getBytes());
+ }
+ out.close();
+ } catch (IOException ex) {
+ countException++;
+ GenericTestUtils.assertExceptionContains("QUOTA_EXCEEDED", ex);
+ }
+ // AllocateBlock failed, bucket usedBytes should be 4 * blockSize
+ Assert.assertEquals(4 * blockSize,
+ store.getVolume(volumeName).getBucket(bucketName).getUsedBytes());
+
+ Assert.assertEquals(7, countException);
}
@Test
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
index 367e4ba..9a7f31a 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
@@ -279,10 +279,11 @@ public class OMFileCreateRequest extends OMKeyRequest {
omVolumeArgs = getVolumeInfo(omMetadataManager, volumeName);
omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName);
- // check volume quota
+ // check bucket and volume quota
long preAllocatedSpace = newLocationList.size()
* ozoneManager.getScmBlockSize()
* omKeyInfo.getFactor().getNumber();
+ checkBucketQuotaInBytes(omBucketInfo, preAllocatedSpace);
checkVolumeQuotaInBytes(omVolumeArgs, preAllocatedSpace);
// Add to cache entry can be done outside of lock for this openKey.
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
index afd6162..194e7ef 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
@@ -196,10 +196,11 @@ public class OMAllocateBlockRequest extends OMKeyRequest {
OmKeyLocationInfo.getFromProtobuf(blockLocation));
omVolumeArgs = getVolumeInfo(omMetadataManager, volumeName);
omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName);
- // check volume quota
+ // check bucket and volume quota
long preAllocatedSpace = newLocationList.size()
* ozoneManager.getScmBlockSize()
* openKeyInfo.getFactor().getNumber();
+ checkBucketQuotaInBytes(omBucketInfo, preAllocatedSpace);
checkVolumeQuotaInBytes(omVolumeArgs, preAllocatedSpace);
// Append new block
openKeyInfo.appendNewBlocks(newLocationList, false);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
index f16153d..5ec79b5 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
@@ -298,7 +298,8 @@ public class OMKeyCreateRequest extends OMKeyRequest {
long preAllocatedSpace = newLocationList.size()
* ozoneManager.getScmBlockSize()
* omKeyInfo.getFactor().getNumber();
- // check volume quota
+ // check bucket and volume quota
+ checkBucketQuotaInBytes(omBucketInfo, preAllocatedSpace);
checkVolumeQuotaInBytes(omVolumeArgs, preAllocatedSpace);
// Add to cache entry can be done outside of lock for this openKey.
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
index e24f4e2..bf99169 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
@@ -556,6 +556,27 @@ public abstract class OMKeyRequest extends OMClientRequest
{
}
/**
+ * Check bucket quota in bytes.
+ * @param omBucketInfo
+ * @param allocateSize
+ * @throws IOException
+ */
+ protected void checkBucketQuotaInBytes(OmBucketInfo omBucketInfo,
+ long allocateSize) throws IOException {
+ if (omBucketInfo.getQuotaInBytes() > OzoneConsts.QUOTA_RESET) {
+ long usedBytes = omBucketInfo.getUsedBytes().sum();
+ long quotaInBytes = omBucketInfo.getQuotaInBytes();
+ if (quotaInBytes - usedBytes < allocateSize) {
+ throw new OMException("The DiskSpace quota of bucket:"
+ + omBucketInfo.getBucketName() + "exceeded: quotaInBytes: "
+ + quotaInBytes + " Bytes but diskspace consumed: " + (usedBytes
+ + allocateSize) + " Bytes.",
+ OMException.ResultCodes.QUOTA_EXCEEDED);
+ }
+ }
+ }
+
+ /**
* Check directory exists. If exists return true, else false.
* @param volumeName
* @param bucketName
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
