This is an automated email from the ASF dual-hosted git repository.
erose pushed a commit to branch HDDS-14496-zdu
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-14496-zdu by this push:
new d04a5e57e06 HDDS-14729. Address followup items in HDDS-14646 (#9838)
d04a5e57e06 is described below
commit d04a5e57e0683d8a41d2f5a83f7de3ada8f615b1
Author: Stephen O'Donnell <[email protected]>
AuthorDate: Mon Mar 2 15:04:27 2026 +0000
HDDS-14729. Address followup items in HDDS-14646 (#9838)
---
.../hdds/scm/server/StorageContainerManager.java | 4 +-
.../scm/server/upgrade/FinalizationManager.java | 2 -
.../server/upgrade/FinalizationManagerImpl.java | 22 +----------
.../upgrade/SCMUpgradeFinalizationContext.java | 43 ----------------------
.../hdds/scm/upgrade/TestScmFinalization.java | 28 +-------------
.../hadoop/hdds/upgrade/TestHddsUpgradeUtils.java | 22 ++---------
6 files changed, 8 insertions(+), 113 deletions(-)
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 32673e87279..373f39c78e4 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -715,7 +715,6 @@ private void initializeSystemManagers(OzoneConfiguration
conf,
finalizationExecutor = new DefaultUpgradeFinalizationExecutor<>();
}
finalizationManager = new FinalizationManagerImpl.Builder()
- .setConfiguration(conf)
.setLayoutVersionManager(scmLayoutVersionManager)
.setStorage(scmStorageConfig)
.setHAManager(scmHAManager)
@@ -788,8 +787,7 @@ private void initializeSystemManagers(OzoneConfiguration
conf,
);
}
- finalizationManager.buildUpgradeContext(scmNodeManager, pipelineManager,
- scmContext);
+ finalizationManager.buildUpgradeContext(scmNodeManager, scmContext);
ReplicationManager.ReplicationManagerConfiguration rmConf =
conf.getObject(ReplicationManager.ReplicationManagerConfiguration.class);
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManager.java
index 421837971bb..27e26541eca 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManager.java
@@ -21,7 +21,6 @@
import java.io.IOException;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.ozone.upgrade.BasicUpgradeFinalizer;
@@ -46,7 +45,6 @@ UpgradeFinalization.StatusAndMessages
queryUpgradeFinalizationProgress(
FinalizationCheckpoint getCheckpoint();
void buildUpgradeContext(NodeManager nodeManager,
- PipelineManager pipelineManager,
SCMContext scmContext);
void reinitialize(Table<String, String> finalizationStore) throws
IOException;
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManagerImpl.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManagerImpl.java
index be65b933a1b..4de0abd7a09 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManagerImpl.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManagerImpl.java
@@ -24,11 +24,9 @@
import java.util.Objects;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
import org.apache.hadoop.hdds.scm.ha.SCMHAManager;
import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
import org.apache.hadoop.hdds.utils.db.Table;
@@ -50,8 +48,6 @@ public class FinalizationManagerImpl implements
FinalizationManager {
private SCMUpgradeFinalizer upgradeFinalizer;
private SCMUpgradeFinalizationContext context;
private SCMStorageConfig storage;
- private OzoneConfiguration conf;
- private HDDSLayoutVersionManager versionManager;
private final FinalizationStateManager finalizationStateManager;
private ThreadFactory threadFactory;
@@ -78,23 +74,16 @@ private FinalizationManagerImpl(Builder builder) throws
IOException {
private void initCommonFields(Builder builder) {
this.storage = builder.storage;
- this.versionManager = builder.versionManager;
- this.conf = builder.conf;
- this.upgradeFinalizer = new SCMUpgradeFinalizer(this.versionManager,
- builder.executor);
+ this.upgradeFinalizer = new SCMUpgradeFinalizer(builder.versionManager,
builder.executor);
}
@Override
public void buildUpgradeContext(NodeManager nodeManager,
- PipelineManager pipelineManager,
- SCMContext scmContext) {
+ SCMContext scmContext) {
this.context = new SCMUpgradeFinalizationContext.Builder()
.setStorage(this.storage)
.setFinalizationStateManager(finalizationStateManager)
- .setConfiguration(conf)
.setNodeManager(nodeManager)
- .setPipelineManager(pipelineManager)
- .setLayoutVersionManager(versionManager)
.setSCMContext(scmContext)
.build();
@@ -172,7 +161,6 @@ public void onLeaderReady() {
* Builds a {@link FinalizationManagerImpl}.
*/
public static class Builder {
- private OzoneConfiguration conf;
private HDDSLayoutVersionManager versionManager;
private SCMStorageConfig storage;
private SCMHAManager scmHAManager;
@@ -183,11 +171,6 @@ public Builder() {
executor = new DefaultUpgradeFinalizationExecutor<>();
}
- public Builder setConfiguration(OzoneConfiguration configuration) {
- this.conf = configuration;
- return this;
- }
-
public Builder setLayoutVersionManager(
HDDSLayoutVersionManager layoutVersionManager) {
this.versionManager = layoutVersionManager;
@@ -217,7 +200,6 @@ public Builder setFinalizationExecutor(
}
public FinalizationManagerImpl build() throws IOException {
- Objects.requireNonNull(conf, "conf == null");
Objects.requireNonNull(versionManager, "versionManager == null");
Objects.requireNonNull(storage, "storage == null");
Objects.requireNonNull(scmHAManager, "scmHAManager == null");
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizationContext.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizationContext.java
index 312f71cb74c..5d5b021f6f0 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizationContext.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizationContext.java
@@ -18,33 +18,24 @@
package org.apache.hadoop.hdds.scm.server.upgrade;
import java.util.Objects;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
-import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
/**
* Provided to methods in the {@link SCMUpgradeFinalizer} to supply objects
* needed to operate.
*/
public final class SCMUpgradeFinalizationContext {
- private final PipelineManager pipelineManager;
private final NodeManager nodeManager;
private final FinalizationStateManager finalizationStateManager;
private final SCMStorageConfig storage;
- private final HDDSLayoutVersionManager versionManager;
- private final OzoneConfiguration conf;
private final SCMContext scmContext;
private SCMUpgradeFinalizationContext(Builder builder) {
- pipelineManager = builder.pipelineManager;
nodeManager = builder.nodeManager;
finalizationStateManager = builder.finalizationStateManager;
storage = builder.storage;
- versionManager = builder.versionManager;
- conf = builder.conf;
scmContext = builder.scmContext;
}
@@ -52,22 +43,10 @@ public NodeManager getNodeManager() {
return nodeManager;
}
- public PipelineManager getPipelineManager() {
- return pipelineManager;
- }
-
public FinalizationStateManager getFinalizationStateManager() {
return finalizationStateManager;
}
- public OzoneConfiguration getConfiguration() {
- return conf;
- }
-
- public HDDSLayoutVersionManager getLayoutVersionManager() {
- return versionManager;
- }
-
public SCMContext getSCMContext() {
return scmContext;
}
@@ -80,12 +59,9 @@ public SCMStorageConfig getStorage() {
* Builds an {@link SCMUpgradeFinalizationContext}.
*/
public static final class Builder {
- private PipelineManager pipelineManager;
private NodeManager nodeManager;
private FinalizationStateManager finalizationStateManager;
private SCMStorageConfig storage;
- private HDDSLayoutVersionManager versionManager;
- private OzoneConfiguration conf;
private SCMContext scmContext;
public Builder() {
@@ -96,11 +72,6 @@ public Builder setSCMContext(SCMContext context) {
return this;
}
- public Builder setPipelineManager(PipelineManager pipelineManager) {
- this.pipelineManager = pipelineManager;
- return this;
- }
-
public Builder setNodeManager(NodeManager nodeManager) {
this.nodeManager = nodeManager;
return this;
@@ -117,24 +88,10 @@ public Builder setStorage(SCMStorageConfig storage) {
return this;
}
- public Builder setLayoutVersionManager(
- HDDSLayoutVersionManager layoutVersionManager) {
- this.versionManager = layoutVersionManager;
- return this;
- }
-
- public Builder setConfiguration(OzoneConfiguration configuration) {
- this.conf = configuration;
- return this;
- }
-
public SCMUpgradeFinalizationContext build() {
Objects.requireNonNull(scmContext, "scmContext == null");
- Objects.requireNonNull(pipelineManager, "pipelineManager == null");
Objects.requireNonNull(nodeManager, "nodeManager == null");
Objects.requireNonNull(storage, "storage == null");
- Objects.requireNonNull(versionManager, "versionManager == null");
- Objects.requireNonNull(conf, "conf == null");
Objects.requireNonNull(finalizationStateManager,
"finalizationStateManager == null");
return new SCMUpgradeFinalizationContext(this);
}
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmFinalization.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmFinalization.java
index c8dce937e41..a6bc7e4acaa 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmFinalization.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmFinalization.java
@@ -20,7 +20,6 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.mockito.Mockito.any;
import static org.mockito.Mockito.eq;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.matches;
@@ -29,16 +28,13 @@
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.when;
-import java.util.Arrays;
import java.util.UUID;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
import org.apache.hadoop.hdds.scm.ha.SCMHAManager;
import org.apache.hadoop.hdds.scm.ha.SCMHAManagerStub;
import org.apache.hadoop.hdds.scm.ha.SCMRatisServer;
import org.apache.hadoop.hdds.scm.metadata.DBTransactionBuffer;
import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationCheckpoint;
import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationManager;
@@ -94,8 +90,6 @@ public void testUpgradeStateToCheckpointMapping() throws
Exception {
HDDSLayoutVersionManager versionManager =
new HDDSLayoutVersionManager(
HDDSLayoutFeature.INITIAL_VERSION.layoutVersion());
- PipelineManager pipelineManager =
- getMockPipelineManager(FinalizationCheckpoint.FINALIZATION_REQUIRED);
// State manager keeps upgrade information in memory as well as writing
// it to disk, so we can mock the classes that handle disk ops for this
// test.
@@ -113,12 +107,9 @@ public void testUpgradeStateToCheckpointMapping() throws
Exception {
stateManager.getFinalizationCheckpoint());
SCMUpgradeFinalizationContext context =
new SCMUpgradeFinalizationContext.Builder()
- .setConfiguration(new OzoneConfiguration())
.setFinalizationStateManager(stateManager)
.setStorage(mock(SCMStorageConfig.class))
- .setLayoutVersionManager(versionManager)
.setSCMContext(scmContext)
- .setPipelineManager(pipelineManager)
.setNodeManager(mock(NodeManager.class))
.build();
stateManager.setUpgradeContext(context);
@@ -201,8 +192,6 @@ public void testResumeFinalizationFromCheckpoint(
SCMStorageConfig storage = mock(SCMStorageConfig.class);
SCMContext scmContext = SCMContext.emptyContext();
scmContext.setFinalizationCheckpoint(initialCheckpoint);
- PipelineManager pipelineManager =
- getMockPipelineManager(initialCheckpoint);
FinalizationStateManager stateManager =
new FinalizationStateManagerTestImpl.Builder()
@@ -214,14 +203,13 @@ public void testResumeFinalizationFromCheckpoint(
FinalizationManager manager = new FinalizationManagerTestImpl.Builder()
.setFinalizationStateManager(stateManager)
- .setConfiguration(new OzoneConfiguration())
.setLayoutVersionManager(versionManager)
.setStorage(storage)
.setHAManager(SCMHAManagerStub.getInstance(true))
.setFinalizationStore(finalizationStore)
.build();
- manager.buildUpgradeContext(nodeManager, pipelineManager, scmContext);
+ manager.buildUpgradeContext(nodeManager, scmContext);
// Execute upgrade finalization, then check that events happened in the
// correct order.
@@ -230,8 +218,7 @@ public void testResumeFinalizationFromCheckpoint(
assertEquals(getStatusFromCheckpoint(initialCheckpoint).status(),
status.status());
- InOrder inOrder = inOrder(buffer, pipelineManager, nodeManager,
- storage);
+ InOrder inOrder = inOrder(buffer, nodeManager, storage);
// Once the initial checkpoint's operations are crossed, this count will
// be increased to 1 to indicate where finalization should have resumed
@@ -325,15 +312,4 @@ private StatusAndMessages getStatusFromCheckpoint(
}
}
- private PipelineManager getMockPipelineManager(
- FinalizationCheckpoint inititalCheckpoint) {
- PipelineManager pipelineManager = mock(PipelineManager.class);
- // After finalization, SCM will wait for at least one pipeline to be
- // created. It does not care about the contents of the pipeline list, so
- // just return something with length >= 1.
- when(pipelineManager.getPipelines(any(),
- any())).thenReturn(Arrays.asList(null, null, null));
-
- return pipelineManager;
- }
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHddsUpgradeUtils.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHddsUpgradeUtils.java
index d9ea4f57f89..facbe3f44d1 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHddsUpgradeUtils.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHddsUpgradeUtils.java
@@ -23,7 +23,6 @@
import static
org.apache.hadoop.ozone.upgrade.UpgradeFinalization.Status.FINALIZATION_DONE;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
@@ -39,7 +38,6 @@
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationCheckpoint;
import org.apache.hadoop.ozone.HddsDatanodeService;
-import org.apache.hadoop.ozone.container.common.interfaces.Container;
import
org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.upgrade.UpgradeFinalization;
import org.apache.ozone.test.GenericTestUtils;
@@ -112,10 +110,7 @@ public static void
testPostUpgradeConditionsSCM(StorageContainerManager scm,
// are ready to serve requests even though containers may remain OPEN.
testDataNodesStateOnSCM(scm, numDatanodes, HEALTHY, HEALTHY_READONLY);
- int countContainers = 0;
- for (ContainerInfo ignored : scm.getContainerManager().getContainers()) {
- countContainers++;
- }
+ int countContainers = scm.getContainerManager().getContainers().size();
assertThat(countContainers).isGreaterThanOrEqualTo(numContainers);
}
@@ -134,13 +129,7 @@ public static void testPreUpgradeConditionsDataNodes(
int countContainers = 0;
for (HddsDatanodeService dataNode : datanodes) {
DatanodeStateMachine dsm = dataNode.getDatanodeStateMachine();
- // Also verify that all the existing containers are open.
- for (Container<?> container :
- dsm.getContainer().getController().getContainers()) {
- assertSame(container.getContainerState(),
- ContainerProtos.ContainerDataProto.State.OPEN);
- countContainers++;
- }
+ countContainers += dsm.getContainer().getContainerSet().containerCount();
}
assertThat(countContainers).isGreaterThanOrEqualTo(1);
}
@@ -179,12 +168,7 @@ public static void testPostUpgradeConditionsDataNodes(
assertEquals(dnVersionManager.getSoftwareLayoutVersion(),
dnVersionManager.getMetadataLayoutVersion());
assertThat(dnVersionManager.getMetadataLayoutVersion()).isGreaterThanOrEqualTo(1);
-
- // Verify containers are in acceptable states (OPEN is now allowed).
- for (Container<?> ignored :
- dsm.getContainer().getController().getContainers()) {
- countContainers++;
- }
+ countContainers += dsm.getContainer().getContainerSet().containerCount();
}
assertThat(countContainers).isGreaterThanOrEqualTo(numContainers);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
