chihsuan commented on code in PR #12053:
URL: https://github.com/apache/ozone/pull/12053#discussion_r4631890467


##########
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java:
##########
@@ -1467,12 +1467,13 @@ public void markContainerForClose(Container container)
           container.getContainerState();
       // Move the container to CLOSING state only if it's OPEN/RECOVERING
       if (HddsUtils.isOpenToWriteState(state)) {
-        if (state == RECOVERING) {
+        boolean wasRecovering = state == RECOVERING;
+        container.markContainerForClose();
+        if (wasRecovering) {
           containerSet.removeRecoveringContainer(
               container.getContainerData().getContainerID());
           ContainerLogger.logRecovered(container.getContainerData());

Review Comment:
   Since `markContainerForClose()` now runs first, this line logs after the 
state is already `CLOSING`. 
   
   So it now prints the same line as `logClosing` below. Can we use something 
like `getMessage(containerData, "Container recovered")` to keep them apart?



##########
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java:
##########
@@ -1467,12 +1467,13 @@ public void markContainerForClose(Container container)
           container.getContainerState();
       // Move the container to CLOSING state only if it's OPEN/RECOVERING
       if (HddsUtils.isOpenToWriteState(state)) {
-        if (state == RECOVERING) {
+        boolean wasRecovering = state == RECOVERING;
+        container.markContainerForClose();
+        if (wasRecovering) {

Review Comment:
   nit: Since state is read before the transition, `wasRecovering` is not 
needed.



##########
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java:
##########
@@ -704,6 +709,101 @@ public void testDeleteRecoveringContainer() throws 
IOException {
     }
   }
 
+  @Test
+  public void testFailedCloseKeepsRecoveringContainerTracked() throws 
IOException {
+    HandlerWithVolumeSet handlerCtx = createKeyValueHandler(tempDir);
+    KeyValueHandler keyValueHandler = handlerCtx.getHandler();
+    ContainerSet containerSet = handlerCtx.getContainerSet();
+    try {
+      ContainerCommandRequestProto.Builder request = 
createContainerRequest(DATANODE_UUID, DUMMY_CONTAINER_ID)
+          .toBuilder();
+      request.getCreateContainerBuilder().setState(State.RECOVERING);
+      assertThat(keyValueHandler.handleCreateContainer(request.build(), 
null).getResult())
+          .isEqualTo(ContainerProtos.Result.SUCCESS);
+      KeyValueContainer container = (KeyValueContainer) 
containerSet.getContainer(DUMMY_CONTAINER_ID);
+      assertThat(container).isNotNull();
+      assertThat(container.getContainerState()).isEqualTo(State.RECOVERING);
+      
assertThat(containerSet.getRecoveringContainerMap()).containsKey(DUMMY_CONTAINER_ID);
+
+      KeyValueContainer containerSpy = spy(container);
+      doThrow(new StorageContainerException("Failed to update .container file",
+          ContainerProtos.Result.CONTAINER_FILES_CREATE_ERROR))
+          .doCallRealMethod()
+          .when(containerSpy).markContainerForClose();
+      StorageContainerException ex = 
assertThrows(StorageContainerException.class,
+          () -> keyValueHandler.markContainerForClose(containerSpy));
+      assertEquals(ContainerProtos.Result.CONTAINER_FILES_CREATE_ERROR, 
ex.getResult());
+      assertThat(container.getContainerState()).isEqualTo(State.RECOVERING);
+      
assertThat(containerSet.getRecoveringContainerMap()).containsKey(DUMMY_CONTAINER_ID);
+
+      keyValueHandler.markContainerForClose(containerSpy);
+      assertThat(container.getContainerState()).isEqualTo(State.CLOSING);
+      
assertThat(containerSet.getRecoveringContainerMap()).doesNotContainKey(DUMMY_CONTAINER_ID);
+    } finally {
+      keyValueHandler.stop();
+      
handlerCtx.getVolumeSet().getVolumesList().forEach(StorageVolume::shutdown);
+    }
+  }
+
+  @Test
+  public void testFailedCloseRecoveringContainerScrubbedToUnhealthy() throws 
Exception {
+    MockClock testClock = new MockClock(Instant.now(), ZoneOffset.UTC);
+    final ContainerSet containerSet = newContainerSet(1000, testClock);
+    final MutableVolumeSet volumeSet = mock(MutableVolumeSet.class);
+
+    HddsVolume hddsVolume = new 
HddsVolume.Builder(tempDir.toString()).conf(conf)
+        .clusterID(CLUSTER_ID).datanodeUuid(DATANODE_UUID)
+        .volumeSet(volumeSet)
+        .build();
+    hddsVolume.format(CLUSTER_ID);
+    hddsVolume.createWorkingDir(CLUSTER_ID, null);
+    hddsVolume.createTmpDirs(CLUSTER_ID);
+    
when(volumeSet.getVolumesList()).thenReturn(Collections.singletonList(hddsVolume));
+    final KeyValueHandler keyValueHandler = 
ContainerTestUtils.getKeyValueHandler(conf,
+        DATANODE_UUID, containerSet, volumeSet);
+    keyValueHandler.setClusterID(CLUSTER_ID);
+    hddsVolume.getVolumeInfoStats().unregister();
+    hddsVolume.getVolumeIOStats().unregister();
+    ContainerMetrics.remove();
+
+    ContainerController controller = new ContainerController(containerSet,
+        Collections.singletonMap(ContainerType.KeyValueContainer, 
keyValueHandler));
+    OnDemandContainerScanner onDemandScanner = new OnDemandContainerScanner(
+        conf.getObject(ContainerScannerConfiguration.class), controller);
+    containerSet.registerOnDemandScanner(onDemandScanner);

Review Comment:
   nit: Could this reuse `createKeyValueHandler`? The setup is the same except 
for the `ContainerSet`, so an overload that accepts a `ContainerSet` may be 
enough.



##########
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java:
##########
@@ -704,6 +709,101 @@ public void testDeleteRecoveringContainer() throws 
IOException {
     }
   }
 
+  @Test
+  public void testFailedCloseKeepsRecoveringContainerTracked() throws 
IOException {
+    HandlerWithVolumeSet handlerCtx = createKeyValueHandler(tempDir);
+    KeyValueHandler keyValueHandler = handlerCtx.getHandler();
+    ContainerSet containerSet = handlerCtx.getContainerSet();
+    try {
+      ContainerCommandRequestProto.Builder request = 
createContainerRequest(DATANODE_UUID, DUMMY_CONTAINER_ID)
+          .toBuilder();
+      request.getCreateContainerBuilder().setState(State.RECOVERING);
+      assertThat(keyValueHandler.handleCreateContainer(request.build(), 
null).getResult())
+          .isEqualTo(ContainerProtos.Result.SUCCESS);
+      KeyValueContainer container = (KeyValueContainer) 
containerSet.getContainer(DUMMY_CONTAINER_ID);
+      assertThat(container).isNotNull();
+      assertThat(container.getContainerState()).isEqualTo(State.RECOVERING);
+      
assertThat(containerSet.getRecoveringContainerMap()).containsKey(DUMMY_CONTAINER_ID);
+
+      KeyValueContainer containerSpy = spy(container);
+      doThrow(new StorageContainerException("Failed to update .container file",
+          ContainerProtos.Result.CONTAINER_FILES_CREATE_ERROR))
+          .doCallRealMethod()
+          .when(containerSpy).markContainerForClose();
+      StorageContainerException ex = 
assertThrows(StorageContainerException.class,
+          () -> keyValueHandler.markContainerForClose(containerSpy));
+      assertEquals(ContainerProtos.Result.CONTAINER_FILES_CREATE_ERROR, 
ex.getResult());
+      assertThat(container.getContainerState()).isEqualTo(State.RECOVERING);

Review Comment:
   Could we make the `.container` write actually fail here instead of stubbing 
it? With the stub, the state rollback never runs, so the `RECOVERING` check 
passes either way.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to