jojochuang commented on code in PR #10217:
URL: https://github.com/apache/ozone/pull/10217#discussion_r3437708764
##########
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestQuotaRepairTask.java:
##########
@@ -187,6 +199,131 @@ public void testQuotaRepairForOldVersionVolumeBucket()
throws Exception {
assertEquals(-1, volArgsVerify.getQuotaInNamespace());
}
+ @Test
+ public void testQuotaRepairDeletedTableSnapshotQuota() throws Exception {
+ OzoneManagerProtocolProtos.OMResponse respMock =
mock(OzoneManagerProtocolProtos.OMResponse.class);
+ when(respMock.getSuccess()).thenReturn(true);
+ OzoneManagerRatisServer ratisServerMock =
mock(OzoneManagerRatisServer.class);
+ AtomicReference<OzoneManagerProtocolProtos.OMRequest> ref = new
AtomicReference<>();
+ doAnswer(invocation -> {
+ ref.set(invocation.getArgument(0,
OzoneManagerProtocolProtos.OMRequest.class));
+ return respMock;
+ }).when(ratisServerMock).submitRequest(any(), any(), anyLong());
+ when(ozoneManager.getOmRatisServer()).thenReturn(ratisServerMock);
+
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager, BucketLayout.OBJECT_STORE);
+
+ String keyName = "/user/snapKey";
+ OMRequestTestUtils.addKeyToTableAndCache(volumeName, bucketName,
+ keyName, -1, RatisReplicationConfig.getInstance(THREE), 1L,
omMetadataManager);
+
+ String ozoneKey = omMetadataManager.getOzoneKey(volumeName, bucketName,
keyName);
+ OmBucketInfo bucketInfo = omMetadataManager.getBucketTable().get(
+ omMetadataManager.getBucketKey(volumeName, bucketName));
+ long bucketObjId = bucketInfo.getObjectID();
+
+ OMRequestTestUtils.deleteKey(ozoneKey, bucketObjId, omMetadataManager, 2L);
+
+ RepeatedOmKeyInfo deletedEntry =
omMetadataManager.getDeletedTable().get(ozoneKey);
+ long expectedSnapNs = deletedEntry.getOmKeyInfoList().size();
+
+ bucketInfo = omMetadataManager.getBucketTable().get(
+ omMetadataManager.getBucketKey(volumeName, bucketName));
+ OmBucketInfo corruptedSnapshot = bucketInfo.toBuilder()
+ .setSnapshotUsedBytes(7L)
+ .setSnapshotUsedNamespace(99L)
+ .build();
+ String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
+ omMetadataManager.getBucketTable().put(bucketKey, corruptedSnapshot);
+ omMetadataManager.getBucketTable().addCacheEntry(
+ new CacheKey<>(bucketKey), CacheValue.get(3L, corruptedSnapshot));
+
+ QuotaRepairTask quotaRepairTask = new QuotaRepairTask(ozoneManager);
+ CompletableFuture<Boolean> repair = quotaRepairTask.repair();
+ assertTrue(awaitRepair(repair));
+
+ OMQuotaRepairRequest omQuotaRepairRequest = new
OMQuotaRepairRequest(ref.get());
+ OMClientResponse omClientResponse =
omQuotaRepairRequest.validateAndUpdateCache(ozoneManager, 1);
+ BatchOperation batchOperation =
omMetadataManager.getStore().initBatchOperation();
+ ((OMQuotaRepairResponse) omClientResponse).addToDBBatch(omMetadataManager,
batchOperation);
+ omMetadataManager.getStore().commitBatchOperation(batchOperation);
+
+ OmBucketInfo repaired = omMetadataManager.getBucketTable().get(bucketKey);
+ assertEquals(0, repaired.getUsedBytes());
+ assertEquals(0, repaired.getUsedNamespace());
+ assertEquals(expectedSnapNs, repaired.getSnapshotUsedNamespace());
+ assertTrue(repaired.getSnapshotUsedBytes() > 0,
+ "Snapshot pending-delete bytes must be recomputed from deletedTable");
+ }
+
+ @Test
+ public void testQuotaRepairSnapshotDbDeletedTableQuota() throws Exception {
+ OzoneManagerProtocolProtos.OMResponse respMock =
mock(OzoneManagerProtocolProtos.OMResponse.class);
+ when(respMock.getSuccess()).thenReturn(true);
+ OzoneManagerRatisServer ratisServerMock =
mock(OzoneManagerRatisServer.class);
+ AtomicReference<OzoneManagerProtocolProtos.OMRequest> ref = new
AtomicReference<>();
+ doAnswer(invocation -> {
+ ref.set(invocation.getArgument(0,
OzoneManagerProtocolProtos.OMRequest.class));
+ return respMock;
+ }).when(ratisServerMock).submitRequest(any(), any(), anyLong());
+ when(ozoneManager.getOmRatisServer()).thenReturn(ratisServerMock);
+
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager, BucketLayout.OBJECT_STORE);
+
+ String keyName = "/user/snapKey";
+ OMRequestTestUtils.addKeyToTableAndCache(volumeName, bucketName,
+ keyName, -1, RatisReplicationConfig.getInstance(THREE), 1L,
omMetadataManager);
+
+ String ozoneKey = omMetadataManager.getOzoneKey(volumeName, bucketName,
keyName);
+ OmKeyInfo omKeyInfo =
omMetadataManager.getKeyTable(BucketLayout.OBJECT_STORE).get(ozoneKey);
+ long keyBytes = omKeyInfo.getReplicatedSize();
+
+ OmBucketInfo bucketInfo = omMetadataManager.getBucketTable().get(
+ omMetadataManager.getBucketKey(volumeName, bucketName));
+ OMRequestTestUtils.deleteKey(ozoneKey, bucketInfo.getObjectID(),
omMetadataManager, 2L);
+
+ String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
+ OmBucketInfo afterDelete = bucketInfo.toBuilder()
+ .setUsedBytes(0)
+ .setUsedNamespace(0)
+ .setSnapshotUsedBytes(keyBytes)
+ .setSnapshotUsedNamespace(1)
+ .build();
+ omMetadataManager.getBucketTable().put(bucketKey, afterDelete);
+
+ when(ozoneManager.getDefaultReplicationConfig())
+ .thenReturn(RatisReplicationConfig.getInstance(THREE));
+ createSnapshot("snap1");
+
+ assertNull(omMetadataManager.getDeletedTable().get(ozoneKey),
+ "Deleted key should move out of active deletedTable after snapshot");
+ assertEquals(0,
omMetadataManager.countRowsInTable(omMetadataManager.getDeletedTable()));
+
+ OmBucketInfo corrupted = afterDelete.toBuilder()
+ .setSnapshotUsedBytes(7L)
+ .build();
+ omMetadataManager.getBucketTable().put(bucketKey, corrupted);
+ omMetadataManager.getBucketTable().addCacheEntry(
+ new CacheKey<>(bucketKey), CacheValue.get(3L, corrupted));
+
+ QuotaRepairTask quotaRepairTask = new QuotaRepairTask(ozoneManager);
+ CompletableFuture<Boolean> repair = quotaRepairTask.repair();
+ assertTrue(awaitRepair(repair));
+
+ OMQuotaRepairRequest omQuotaRepairRequest = new
OMQuotaRepairRequest(ref.get());
+ OMClientResponse omClientResponse =
omQuotaRepairRequest.validateAndUpdateCache(ozoneManager, 1);
+ BatchOperation batchOperation =
omMetadataManager.getStore().initBatchOperation();
+ ((OMQuotaRepairResponse) omClientResponse).addToDBBatch(omMetadataManager,
batchOperation);
+ omMetadataManager.getStore().commitBatchOperation(batchOperation);
+
+ OmBucketInfo repaired = omMetadataManager.getBucketTable().get(bucketKey);
+ assertEquals(0, repaired.getUsedBytes());
+ assertEquals(0, repaired.getUsedNamespace());
+ assertEquals(keyBytes, repaired.getSnapshotUsedBytes());
Review Comment:
repaired.getSnapshotUsedNamespace() not checked?
##########
hadoop-ozone/interface-client/src/main/resources/proto.lock:
##########
@@ -8515,6 +8515,18 @@
"value": "false"
}
]
+ },
Review Comment:
this is actually not needed right now. We usually update proto.lock when
release a new version.
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/QuotaRepairTask.java:
##########
@@ -311,6 +345,24 @@ private void repairCount(
tasks.add(executor.submit(() -> recalculateUsages(
metadataManager.getDirectoryTable(),
directoryCountMap, "Directory usages", false)));
+
+ Map<Long, OmBucketInfo> bucketById =
buildBucketByObjectId(idBucketInfoMap);
+
+ tasks.add(executor.submit(() -> recalculateSnapshotDeletedKeyUsages(
Review Comment:
the method name recalculateSnapshotDeletedKeyUsages() is confusing. It is
applied to both AOS and snapshot db.
Same for recalculateSnapshotDeletedDirNamespace().
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
