ashishkumar50 commented on code in PR #4655:
URL: https://github.com/apache/ozone/pull/4655#discussion_r1212757648
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java:
##########
@@ -435,6 +441,257 @@ public void testContainerStatisticsAfterDelete() throws
Exception {
LOG.info(metrics.toString());
}
+ @Test
+ public void testContainerStateAfterDNRestart() throws Exception {
+ ReplicationManager replicationManager = scm.getReplicationManager();
+
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+
+ String value = RandomStringUtils.random(10 * 10);
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ String keyName = UUID.randomUUID().toString();
+ OzoneOutputStream out = bucket.createKey(keyName,
+ value.getBytes(UTF_8).length, ReplicationType.RATIS,
+ ReplicationFactor.THREE, new HashMap<>());
+ out.write(value.getBytes(UTF_8));
+ out.close();
+
+ OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName)
+ .setBucketName(bucketName).setKeyName(keyName).setDataSize(0)
+ .setReplicationConfig(
+ RatisReplicationConfig
+ .getInstance(HddsProtos.ReplicationFactor.THREE))
+ .build();
+ List<OmKeyLocationInfoGroup> omKeyLocationInfoGroupList =
+ om.lookupKey(keyArgs).getKeyLocationVersions();
+ Thread.sleep(5000);
+ List<ContainerInfo> containerInfos =
+ scm.getContainerManager().getContainers();
+ final int valueSize = value.getBytes(UTF_8).length;
+ final int keyCount = 1;
+ containerInfos.stream().forEach(container -> {
+ Assertions.assertEquals(valueSize, container.getUsedBytes());
+ Assertions.assertEquals(keyCount, container.getNumberOfKeys());
+ });
+
+ OzoneTestUtils.closeAllContainers(scm.getEventQueue(), scm);
+ // Wait for container to close
+ Thread.sleep(2000);
+ // make sure the containers are closed on the dn
+ omKeyLocationInfoGroupList.forEach((group) -> {
+ List<OmKeyLocationInfo> locationInfo = group.getLocationList();
+ locationInfo.forEach(
+ (info) -> cluster.getHddsDatanodes().get(0).getDatanodeStateMachine()
+ .getContainer().getContainerSet()
+ .getContainer(info.getContainerID()).getContainerData()
+ .setState(ContainerProtos.ContainerDataProto.State.CLOSED));
+ });
+
+ ContainerID containerId = ContainerID.valueOf(
+ containerInfos.get(0).getContainerID());
+ // Before restart container state is non-empty
+ Assertions.assertFalse(getContainerFromDN(
+ cluster.getHddsDatanodes().get(0), containerId.getId())
+ .getContainerData().isEmpty());
+ // Restart DataNode
+ cluster.restartHddsDatanode(0, true);
+
+ // After restart also container state remains non-empty.
+ Assertions.assertFalse(getContainerFromDN(
+ cluster.getHddsDatanodes().get(0), containerId.getId())
+ .getContainerData().isEmpty());
+
+ // Delete key
+ writeClient.deleteKey(keyArgs);
+ Thread.sleep(10000);
+
+ GenericTestUtils.waitFor(() -> {
+ try {
+ return scm.getContainerManager().getContainerReplicas(
+ containerId).stream().
+ allMatch(replica -> replica.isEmpty());
+ } catch (ContainerNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ },
+ 100, 10 * 1000);
+
+ // Container state should be empty now as key got deleted
+ Assertions.assertTrue(getContainerFromDN(
+ cluster.getHddsDatanodes().get(0), containerId.getId())
+ .getContainerData().isEmpty());
+
+ // Restart DataNode
+ cluster.restartHddsDatanode(0, true);
+ // Container state should be empty even after restart
+ Assertions.assertTrue(getContainerFromDN(
+ cluster.getHddsDatanodes().get(0), containerId.getId())
+ .getContainerData().isEmpty());
+
+ GenericTestUtils.waitFor(() -> {
+ replicationManager.processAll();
+ ((EventQueue)scm.getEventQueue()).processAll(1000);
+ List<ContainerInfo> infos = scm.getContainerManager().getContainers();
+ try {
+ infos.stream().forEach(container -> {
+ Assertions.assertEquals(HddsProtos.LifeCycleState.DELETED,
+ container.getState());
+ try {
+ Assertions.assertEquals(HddsProtos.LifeCycleState.DELETED,
+ scm.getScmMetadataStore().getContainerTable()
+ .get(container.containerID()).getState());
+ } catch (IOException e) {
+ Assertions.fail(
+ "Container from SCM DB should be marked as DELETED");
+ }
+ });
+ } catch (Throwable e) {
+ LOG.info(e.getMessage());
+ return false;
+ }
+ return true;
+ }, 500, 30000);
+ LOG.info(metrics.toString());
+ }
+
+ /**
+ * Return the container for the given containerID from the given DN.
+ */
+ private Container getContainerFromDN(HddsDatanodeService hddsDatanodeService,
+ long containerID) {
+ return hddsDatanodeService.getDatanodeStateMachine().getContainer()
+ .getContainerSet().getContainer(containerID);
+ }
+
+ @Test
+ public void testContainerDeleteWithInvalidKeyCount()
+ throws Exception {
+ ReplicationManager replicationManager = scm.getReplicationManager();
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+
+ String value = RandomStringUtils.random(1024 * 1024);
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ String keyName = UUID.randomUUID().toString();
+ OzoneOutputStream out = bucket.createKey(keyName,
+ value.getBytes(UTF_8).length, ReplicationType.RATIS,
+ ReplicationFactor.THREE, new HashMap<>());
+ out.write(value.getBytes(UTF_8));
+ out.close();
+
+ OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName)
+ .setBucketName(bucketName).setKeyName(keyName).setDataSize(0)
+ .setReplicationConfig(
+ RatisReplicationConfig
+ .getInstance(HddsProtos.ReplicationFactor.THREE))
+ .build();
+ List<OmKeyLocationInfoGroup> omKeyLocationInfoGroupList =
+ om.lookupKey(keyArgs).getKeyLocationVersions();
+ Thread.sleep(5000);
+ List<ContainerInfo> containerInfos =
+ scm.getContainerManager().getContainers();
+ final int valueSize = value.getBytes(UTF_8).length;
+ final int keyCount = 1;
+ containerInfos.stream().forEach(container -> {
+ Assertions.assertEquals(valueSize, container.getUsedBytes());
+ Assertions.assertEquals(keyCount, container.getNumberOfKeys());
+ });
+
+ OzoneTestUtils.closeAllContainers(scm.getEventQueue(), scm);
+ // Wait for container to close
+ Thread.sleep(2000);
+ // make sure the containers are closed on the dn
+ omKeyLocationInfoGroupList.forEach((group) -> {
+ List<OmKeyLocationInfo> locationInfo = group.getLocationList();
+ locationInfo.forEach(
+ (info) -> cluster.getHddsDatanodes().get(0).getDatanodeStateMachine()
+ .getContainer().getContainerSet()
+ .getContainer(info.getContainerID()).getContainerData()
+ .setState(ContainerProtos.ContainerDataProto.State.CLOSED));
+ });
+
+ ContainerStateManager containerStateManager = scm.getContainerManager()
+ .getContainerStateManager();
+ ContainerID containerId = ContainerID.valueOf(
+ containerInfos.get(0).getContainerID());
+ // Get all the replicas state from SCM
+ Set<ContainerReplica> replicas
+ = scm.getContainerManager().getContainerReplicas(containerId);
+
+ // Ensure for all replica isEmpty are false in SCM
+ Assert.assertTrue(scm.getContainerManager().getContainerReplicas(
+ containerId).stream().
+ allMatch(replica -> !replica.isEmpty()));
+
+ // Delete key
+ writeClient.deleteKey(keyArgs);
+ Thread.sleep(5000);
+
+ // Ensure isEmpty are true for all replica after delete key
+ GenericTestUtils.waitFor(() -> {
+ try {
+ return scm.getContainerManager().getContainerReplicas(
+ containerId).stream()
+ .allMatch(replica -> replica.isEmpty());
+ } catch (ContainerNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ },
+ 500, 5 * 2000);
Review Comment:
Added high time interval for RM.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
