[
https://issues.apache.org/jira/browse/HDFS-17642?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17895402#comment-17895402
]
ASF GitHub Bot commented on HDFS-17642:
---------------------------------------
ctrezzo commented on code in PR #7127:
URL: https://github.com/apache/hadoop/pull/7127#discussion_r1828252723
##########
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java:
##########
@@ -1932,6 +2083,114 @@ void testBalancerRPCDelay(int getBlocksMaxQps) throws
Exception {
getBlocksMaxQps, getBlockCallsPerSecond <= getBlocksMaxQps);
}
+ /**
+ * Test balancer with excluded target nodes.
+ */
+ @Test(timeout=100000)
+ public void testBalancerExcludeTargetNodes() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ Set<String> excludeTargetNodes = new HashSet<>();
+ excludeTargetNodes.add("datanodeZ");
+ BalancerParameters.Builder pBuilder = new BalancerParameters.Builder();
+ pBuilder.setExcludedTargetNodes(excludeTargetNodes);
+ BalancerParameters p = pBuilder.build();
+ try {
+ doTest(conf, CAPACITY, RACK2,
+ new HostNameBasedNodes(new String[]{"datanodeX", "datanodeY",
"datanodeZ"},
+ BalancerParameters.DEFAULT.getExcludedNodes(),
+ BalancerParameters.DEFAULT.getIncludedNodes()), false,
+ false, p);
+ } catch (Exception e) {
+ if
(!e.getMessage().contains(String.valueOf(ExitStatus.NO_MOVE_BLOCK.getExitCode())))
{
+ throw e;
+ }
+ }
+ }
+
+ /**
+ * Test balancer with included target nodes.
+ */
+ @Test(timeout=100000)
+ public void testBalancerIncludeTargetNodes() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ Set<String> includeTargetNodes = new HashSet<>();
+ includeTargetNodes.add("datanodeY");
+ includeTargetNodes.add("datanodeZ");
+ BalancerParameters.Builder pBuilder = new BalancerParameters.Builder();
+ pBuilder.setTargetNodes(includeTargetNodes);
+ BalancerParameters p = pBuilder.build();
+ try {
+ doTest(conf, CAPACITY, RACK2,
+ new HostNameBasedNodes(new String[]{"datanodeX", "datanodeY",
"datanodeZ"},
+ BalancerParameters.DEFAULT.getExcludedNodes(),
+ BalancerParameters.DEFAULT.getIncludedNodes()), false,
+ false, p);
+ } catch (Exception e) {
+ if
(!e.getMessage().contains(String.valueOf(ExitStatus.NO_MOVE_BLOCK.getExitCode())))
{
+ throw e;
+ }
+ }
+ }
+
+ /**
+ * Test balancer with included source nodes.
+ * Since newly added nodes are the only included source nodes no balancing
will occur.
+ */
+ @Test(timeout=100000)
+ public void testBalancerIncludeSourceNodes() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ Set<String> includeSourceNodes = new HashSet<>();
+ includeSourceNodes.add("datanodeX");
+ includeSourceNodes.add("datanodeY");
+ includeSourceNodes.add("datanodeZ");
+ BalancerParameters.Builder pBuilder = new BalancerParameters.Builder();
+ pBuilder.setSourceNodes(includeSourceNodes);
+ BalancerParameters p = pBuilder.build();
+ try {
+ doTest(conf, CAPACITY, RACK2,
+ new HostNameBasedNodes(new String[]{"datanodeX", "datanodeY",
"datanodeZ"},
+ BalancerParameters.DEFAULT.getExcludedNodes(),
+ BalancerParameters.DEFAULT.getIncludedNodes()), false,
+ false, p);
+ } catch (Exception e) {
+ if
(!e.getMessage().contains(String.valueOf(ExitStatus.NO_MOVE_BLOCK.getExitCode())))
{
+ throw e;
+ }
+ }
+ }
+
+ /**
+ * Test balancer with excluded source nodes.
+ * Since newly added nodes will not be selected as a source,
+ * all nodes will be included in balancing.
+ */
+ @Test(timeout=100000)
+ public void testBalancerExcludeSourceNodes() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ Set<String> excludeSourceNodes = new HashSet<>();
+ excludeSourceNodes.add("datanodeX");
+ excludeSourceNodes.add("datanodeY");
+ BalancerParameters.Builder pBuilder = new BalancerParameters.Builder();
+ pBuilder.setExcludedSourceNodes(excludeSourceNodes);
+ BalancerParameters p = pBuilder.build();
+ try {
+ doTest(conf, CAPACITY, RACK2,
+ new HostNameBasedNodes(new String[]{"datanodeX", "datanodeY",
"datanodeZ"},
+ BalancerParameters.DEFAULT.getExcludedNodes(),
+ BalancerParameters.DEFAULT.getIncludedNodes()), false,
+ false, p);
+ } catch (Exception e) {
+ if
(!e.getMessage().contains(String.valueOf(ExitStatus.NO_MOVE_BLOCK.getExitCode())))
{
Review Comment:
@Jtdellaringa
Right now the unit tests pass in the cases where:
1. no exception is thrown
2. an exception with NO_MOVE_BLOCK exit code in the message
Are the above two cases intended behavior?
If #2 is the only expected pass case, then we should either annotate the
test with the expected failure or use the junit assertThrows (I forget which
version this assert is available in).
> Support specifying Datanodes to exclude for balancing at source/target
> granularity
> ----------------------------------------------------------------------------------
>
> Key: HDFS-17642
> URL: https://issues.apache.org/jira/browse/HDFS-17642
> Project: Hadoop HDFS
> Issue Type: Improvement
> Components: balancer & mover
> Reporter: Joseph Dell'Aringa
> Assignee: Joseph Dell'Aringa
> Priority: Major
> Labels: pull-request-available
>
> In some cases it can be useful to exclude a list of datanodes from being
> selected as a target but remaining an option for a source and vice-versa.
> The fix for this ticket will add an additional exclude list for source and
> target datanodes.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: hdfs-issues-h...@hadoop.apache.org
