chihsuan commented on code in PR #12053:
URL: https://github.com/apache/ozone/pull/12053#discussion_r4631890467
##########
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java:
##########
@@ -1467,12 +1467,13 @@ public void markContainerForClose(Container container)
container.getContainerState();
// Move the container to CLOSING state only if it's OPEN/RECOVERING
if (HddsUtils.isOpenToWriteState(state)) {
- if (state == RECOVERING) {
+ boolean wasRecovering = state == RECOVERING;
+ container.markContainerForClose();
+ if (wasRecovering) {
containerSet.removeRecoveringContainer(
container.getContainerData().getContainerID());
ContainerLogger.logRecovered(container.getContainerData());
Review Comment:
Since `markContainerForClose()` now runs first, this line logs after the
state is already `CLOSING`.
So it now prints the same line as `logClosing` below. Can we use something
like `getMessage(containerData, "Container recovered")` to keep them apart?
##########
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java:
##########
@@ -1467,12 +1467,13 @@ public void markContainerForClose(Container container)
container.getContainerState();
// Move the container to CLOSING state only if it's OPEN/RECOVERING
if (HddsUtils.isOpenToWriteState(state)) {
- if (state == RECOVERING) {
+ boolean wasRecovering = state == RECOVERING;
+ container.markContainerForClose();
+ if (wasRecovering) {
Review Comment:
nit: Since state is read before the transition, `wasRecovering` is not
needed.
##########
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java:
##########
@@ -704,6 +709,101 @@ public void testDeleteRecoveringContainer() throws
IOException {
}
}
+ @Test
+ public void testFailedCloseKeepsRecoveringContainerTracked() throws
IOException {
+ HandlerWithVolumeSet handlerCtx = createKeyValueHandler(tempDir);
+ KeyValueHandler keyValueHandler = handlerCtx.getHandler();
+ ContainerSet containerSet = handlerCtx.getContainerSet();
+ try {
+ ContainerCommandRequestProto.Builder request =
createContainerRequest(DATANODE_UUID, DUMMY_CONTAINER_ID)
+ .toBuilder();
+ request.getCreateContainerBuilder().setState(State.RECOVERING);
+ assertThat(keyValueHandler.handleCreateContainer(request.build(),
null).getResult())
+ .isEqualTo(ContainerProtos.Result.SUCCESS);
+ KeyValueContainer container = (KeyValueContainer)
containerSet.getContainer(DUMMY_CONTAINER_ID);
+ assertThat(container).isNotNull();
+ assertThat(container.getContainerState()).isEqualTo(State.RECOVERING);
+
assertThat(containerSet.getRecoveringContainerMap()).containsKey(DUMMY_CONTAINER_ID);
+
+ KeyValueContainer containerSpy = spy(container);
+ doThrow(new StorageContainerException("Failed to update .container file",
+ ContainerProtos.Result.CONTAINER_FILES_CREATE_ERROR))
+ .doCallRealMethod()
+ .when(containerSpy).markContainerForClose();
+ StorageContainerException ex =
assertThrows(StorageContainerException.class,
+ () -> keyValueHandler.markContainerForClose(containerSpy));
+ assertEquals(ContainerProtos.Result.CONTAINER_FILES_CREATE_ERROR,
ex.getResult());
+ assertThat(container.getContainerState()).isEqualTo(State.RECOVERING);
+
assertThat(containerSet.getRecoveringContainerMap()).containsKey(DUMMY_CONTAINER_ID);
+
+ keyValueHandler.markContainerForClose(containerSpy);
+ assertThat(container.getContainerState()).isEqualTo(State.CLOSING);
+
assertThat(containerSet.getRecoveringContainerMap()).doesNotContainKey(DUMMY_CONTAINER_ID);
+ } finally {
+ keyValueHandler.stop();
+
handlerCtx.getVolumeSet().getVolumesList().forEach(StorageVolume::shutdown);
+ }
+ }
+
+ @Test
+ public void testFailedCloseRecoveringContainerScrubbedToUnhealthy() throws
Exception {
+ MockClock testClock = new MockClock(Instant.now(), ZoneOffset.UTC);
+ final ContainerSet containerSet = newContainerSet(1000, testClock);
+ final MutableVolumeSet volumeSet = mock(MutableVolumeSet.class);
+
+ HddsVolume hddsVolume = new
HddsVolume.Builder(tempDir.toString()).conf(conf)
+ .clusterID(CLUSTER_ID).datanodeUuid(DATANODE_UUID)
+ .volumeSet(volumeSet)
+ .build();
+ hddsVolume.format(CLUSTER_ID);
+ hddsVolume.createWorkingDir(CLUSTER_ID, null);
+ hddsVolume.createTmpDirs(CLUSTER_ID);
+
when(volumeSet.getVolumesList()).thenReturn(Collections.singletonList(hddsVolume));
+ final KeyValueHandler keyValueHandler =
ContainerTestUtils.getKeyValueHandler(conf,
+ DATANODE_UUID, containerSet, volumeSet);
+ keyValueHandler.setClusterID(CLUSTER_ID);
+ hddsVolume.getVolumeInfoStats().unregister();
+ hddsVolume.getVolumeIOStats().unregister();
+ ContainerMetrics.remove();
+
+ ContainerController controller = new ContainerController(containerSet,
+ Collections.singletonMap(ContainerType.KeyValueContainer,
keyValueHandler));
+ OnDemandContainerScanner onDemandScanner = new OnDemandContainerScanner(
+ conf.getObject(ContainerScannerConfiguration.class), controller);
+ containerSet.registerOnDemandScanner(onDemandScanner);
Review Comment:
nit: Could this reuse `createKeyValueHandler`? The setup is the same except
for the `ContainerSet`, so an overload that accepts a `ContainerSet` may be
enough.
##########
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java:
##########
@@ -704,6 +709,101 @@ public void testDeleteRecoveringContainer() throws
IOException {
}
}
+ @Test
+ public void testFailedCloseKeepsRecoveringContainerTracked() throws
IOException {
+ HandlerWithVolumeSet handlerCtx = createKeyValueHandler(tempDir);
+ KeyValueHandler keyValueHandler = handlerCtx.getHandler();
+ ContainerSet containerSet = handlerCtx.getContainerSet();
+ try {
+ ContainerCommandRequestProto.Builder request =
createContainerRequest(DATANODE_UUID, DUMMY_CONTAINER_ID)
+ .toBuilder();
+ request.getCreateContainerBuilder().setState(State.RECOVERING);
+ assertThat(keyValueHandler.handleCreateContainer(request.build(),
null).getResult())
+ .isEqualTo(ContainerProtos.Result.SUCCESS);
+ KeyValueContainer container = (KeyValueContainer)
containerSet.getContainer(DUMMY_CONTAINER_ID);
+ assertThat(container).isNotNull();
+ assertThat(container.getContainerState()).isEqualTo(State.RECOVERING);
+
assertThat(containerSet.getRecoveringContainerMap()).containsKey(DUMMY_CONTAINER_ID);
+
+ KeyValueContainer containerSpy = spy(container);
+ doThrow(new StorageContainerException("Failed to update .container file",
+ ContainerProtos.Result.CONTAINER_FILES_CREATE_ERROR))
+ .doCallRealMethod()
+ .when(containerSpy).markContainerForClose();
+ StorageContainerException ex =
assertThrows(StorageContainerException.class,
+ () -> keyValueHandler.markContainerForClose(containerSpy));
+ assertEquals(ContainerProtos.Result.CONTAINER_FILES_CREATE_ERROR,
ex.getResult());
+ assertThat(container.getContainerState()).isEqualTo(State.RECOVERING);
Review Comment:
Could we make the `.container` write actually fail here instead of stubbing
it? With the stub, the state rollback never runs, so the `RECOVERING` check
passes either way.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]