sarvekshayr commented on code in PR #10812:
URL: https://github.com/apache/ozone/pull/10812#discussion_r3654950851
##########
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java:
##########
@@ -371,6 +383,84 @@ public void testGetBalancerStatusInfoAfterScmStop() throws
Exception {
assertNotNull(statusInfo.getStoppedAt());
}
+ /**
+ * Tests new startup validation for conflicting include/exclude lists.
+ */
+ @Test
+ public void testRejectConflictingIncludeExcludeLists() throws Exception {
+ when(nodeManager.getNodesByAddress("dn0"))
+ .thenReturn(Collections.singletonList(mock(DatanodeDetails.class)));
+ when(nodeManager.getNodesByAddress("dn1"))
+ .thenReturn(Collections.singletonList(mock(DatanodeDetails.class)));
+ balancerConfiguration.setIncludeNodes("dn0,dn1");
+ balancerConfiguration.setExcludeNodes("dn0,dn1");
+
+ InvalidContainerBalancerConfigurationException ex =
+ assertThrows(InvalidContainerBalancerConfigurationException.class,
+ () -> containerBalancer.startBalancer(balancerConfiguration));
+ assertThat(ex.getMessage()).contains(
+ "include-datanodes is a subset of exclude-datanodes");
+ assertSame(ContainerBalancerTask.Status.STOPPED,
containerBalancer.getBalancerStatus());
+
+ balancerConfiguration.setIncludeNodes("");
+ balancerConfiguration.setExcludeNodes("");
+ balancerConfiguration.setIncludeContainers("1, 2");
+ balancerConfiguration.setExcludeContainers("1,2,3");
+ ex = assertThrows(InvalidContainerBalancerConfigurationException.class,
+ () -> containerBalancer.startBalancer(balancerConfiguration));
+ assertThat(ex.getMessage()).contains(
+ "include-containers is a subset of exclude-containers");
+ assertSame(ContainerBalancerTask.Status.STOPPED,
containerBalancer.getBalancerStatus());
+ }
+
+ /**
+ * Tests new startup validation for include-containers, datanode pool, and
+ * size limits.
+ */
+ @Test
+ public void testRejectInvalidStartupConfiguration() throws Exception {
+ ContainerManager containerManager = mock(ContainerManager.class);
+ when(scm.getContainerManager()).thenReturn(containerManager);
+ when(containerManager.getContainer(any(ContainerID.class)))
+ .thenThrow(new ContainerNotFoundException(ContainerID.valueOf(1)));
+
+ balancerConfiguration.setIncludeContainers("1");
+ InvalidContainerBalancerConfigurationException ex =
+ assertThrows(InvalidContainerBalancerConfigurationException.class,
+ () -> containerBalancer.startBalancer(balancerConfiguration));
+ assertThat(ex.getMessage()).contains("do not exist in SCM");
+ assertSame(ContainerBalancerTask.Status.STOPPED,
containerBalancer.getBalancerStatus());
+
+ balancerConfiguration.setIncludeContainers("");
+ List<DatanodeInfo> fiveDatanodes = createEligibleDatanodes(5);
+ when(nodeManager.getNodes(IN_SERVICE, HEALTHY)).thenReturn(fiveDatanodes);
+ balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(20);
+ ex = assertThrows(InvalidContainerBalancerConfigurationException.class,
+ () -> containerBalancer.startBalancer(balancerConfiguration));
+ assertThat(ex.getMessage()).contains("at least 2 are required for a source
and target datanode pair.");
+ assertSame(ContainerBalancerTask.Status.STOPPED,
containerBalancer.getBalancerStatus());
+
+ balancerConfiguration.setMaxSizeToMovePerIteration(100 * OzoneConsts.GB);
+ balancerConfiguration.setMaxSizeEnteringTarget(200 * OzoneConsts.GB);
+ ex = assertThrows(InvalidContainerBalancerConfigurationException.class,
+ () -> containerBalancer.startBalancer(balancerConfiguration));
+ assertThat(ex.getMessage()).contains(
Review Comment:
Add a test to validate the below condition as well -
```
if (conf.getMaxSizeLeavingSource() >
conf.getMaxSizeToMovePerIteration()) {
LOG.warn("hdds.container.balancer.size.leaving.source.max {} should be
"
+ "less than or equal to
hdds.container.balancer.size.moved.max"
+ ".per.iteration {}",
conf.getMaxSizeLeavingSource(),
conf.getMaxSizeToMovePerIteration());
throw new InvalidContainerBalancerConfigurationException(
"hdds.container.balancer.size.leaving.source.max should be less "
+ "than or equal to hdds.container.balancer.size.moved.max.per"
+ ".iteration");
}
```
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java:
##########
@@ -510,6 +541,133 @@ private void
validateConfiguration(ContainerBalancerConfiguration conf)
validateNodeList(conf.getIncludeNodes(), "included");
validateNodeList(conf.getExcludeNodes(), "excluded");
+ validateIncludeExcludeLists(conf);
+ validateIncludeContainersExist(conf);
+ validateEligibleDatanodePool(conf);
+ }
+
+ /**
+ * Rejects include lists that are fully covered by the corresponding exclude
+ * lists, which would leave no datanodes or containers to balance.
+ */
+ private void validateIncludeExcludeLists(ContainerBalancerConfiguration conf)
+ throws InvalidContainerBalancerConfigurationException {
+ Set<String> includeNodes = conf.getIncludeNodes();
+ Set<String> excludeNodes = conf.getExcludeNodes();
+ if (!includeNodes.isEmpty() && !excludeNodes.isEmpty()) {
+ boolean allIncludedNodesExcluded = true;
+ for (String includedNode : includeNodes) {
+ if (!isIncludedNodeExcluded(includedNode, excludeNodes)) {
+ allIncludedNodesExcluded = false;
+ break;
+ }
+ }
+ if (allIncludedNodesExcluded) {
+ throw new InvalidContainerBalancerConfigurationException(
+ "include-datanodes is a subset of exclude-datanodes, no datanode
can participate in balancing.");
+ }
+ }
+
+ Set<ContainerID> includeContainers = conf.getIncludeContainers();
+ Set<ContainerID> excludeContainers = conf.getExcludeContainers();
+ if (!includeContainers.isEmpty() &&
excludeContainers.containsAll(includeContainers)) {
+ throw new InvalidContainerBalancerConfigurationException(
+ "include-containers is a subset of exclude-containers, no container
can be selected for balancing.");
+ }
+ }
+
+ private boolean isIncludedNodeExcluded(String includedNode, Set<String>
excludeNodes) {
+ if (excludeNodes.contains(includedNode)) {
+ return true;
+ }
+ for (DatanodeDetails dn :
scm.getScmNodeManager().getNodesByAddress(includedNode)) {
+ if (excludeNodes.contains(dn.getHostName()) ||
excludeNodes.contains(dn.getIpAddress())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Rejects non-empty include-containers lists when any listed container ID
+ * does not exist in SCM.
+ */
+ private void validateIncludeContainersExist(ContainerBalancerConfiguration
conf)
+ throws InvalidContainerBalancerConfigurationException {
+ Set<ContainerID> includeContainers = conf.getIncludeContainers();
+ if (includeContainers.isEmpty()) {
+ return;
+ }
+
+ ContainerManager containerManager = scm.getContainerManager();
+ List<ContainerID> missingContainers = new ArrayList<>();
+ for (ContainerID containerID : includeContainers) {
+ try {
+ containerManager.getContainer(containerID);
+ } catch (ContainerNotFoundException e) {
+ missingContainers.add(containerID);
+ }
+ }
+
+ if (!missingContainers.isEmpty()) {
+ throw new InvalidContainerBalancerConfigurationException(
+ "Container Balancer cannot start: included container ID(s) " +
missingContainers
+ + " do not exist in SCM.");
+ }
+ }
+
+ /**
+ * Validates that enough healthy, in-service datanodes are eligible and that
+ * {@link
ContainerBalancerConfiguration#getMaxDatanodesRatioToInvolvePerIteration()}
+ * allows at least one source and one target datanode per iteration.
+ */
+ private void validateEligibleDatanodePool(ContainerBalancerConfiguration
conf)
+ throws InvalidContainerBalancerConfigurationException {
+ int eligibleCount = countEligibleDatanodes(conf);
+ if (eligibleCount < 2) {
+ throw new InvalidContainerBalancerConfigurationException(String.format(
+ "Container Balancer requires at least 2 eligible datanodes but only "
+ + "%d is available.", eligibleCount));
Review Comment:
For better readability -
```suggestion
"Container Balancer found %d eligible datanode(s) but requires at
least 2.", eligibleCount));
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
