This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch branch-3 in repository https://gitbox.apache.org/repos/asf/hbase.git
commit e4e93a269dd911a56cfe76c8a267ee6c5ef42784 Author: Liu Xiao <[email protected]> AuthorDate: Tue Feb 24 22:16:54 2026 +0800 HBASE-28913 LoadBalancerPerformanceEvaluation fails with NPE (#7745) Signed-off-by: Duo Zhang <[email protected]> (cherry picked from commit a444fc509c3dfcbfddabba4236e82815512ecfab) --- .../master/balancer/StochasticLoadBalancer.java | 8 +- .../LoadBalancerPerformanceEvaluation.java | 121 ++++++++++++++++++++- .../TestLoadBalancerPerformanceEvaluation.java | 103 ++++++++++++++++++ 3 files changed, 222 insertions(+), 10 deletions(-) diff --git a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index 075d970fc69..0eedc8fc5c5 100644 --- a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -291,10 +291,10 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { curFunctionCosts = new double[costFunctions.size()]; tempFunctionCosts = new double[costFunctions.size()]; - LOG.info("Loaded config; maxSteps=" + maxSteps + ", runMaxSteps=" + runMaxSteps - + ", stepsPerRegion=" + stepsPerRegion + ", maxRunningTime=" + maxRunningTime + ", isByTable=" - + isByTable + ", CostFunctions=" + Arrays.toString(getCostFunctionNames()) - + " , sum of multiplier of cost functions = " + sumMultiplier + " etc."); + LOG.info( + "Loaded config: maxSteps={}, runMaxSteps={}, stepsPerRegion={}, maxRunningTime={}, isByTable={}, CostFunctions={}, sum of multiplier of cost functions = {} etc.", + maxSteps, runMaxSteps, stepsPerRegion, maxRunningTime, isByTable, + Arrays.toString(getCostFunctionNames()), sumMultiplier); } @Override diff --git a/hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerPerformanceEvaluation.java b/hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerPerformanceEvaluation.java index 0e3977dc31a..5c8adeedf46 100644 --- a/hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerPerformanceEvaluation.java +++ b/hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerPerformanceEvaluation.java @@ -19,18 +19,28 @@ package org.apache.hadoop.hbase.master.balancer; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; +import java.util.function.Supplier; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HDFSBlocksDistribution; +import org.apache.hadoop.hbase.ServerMetrics; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.BalancerDecision; +import org.apache.hadoop.hbase.client.BalancerRejection; +import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.master.LoadBalancer; import org.apache.hadoop.hbase.util.AbstractHBaseTool; import org.apache.hadoop.hbase.util.Bytes; @@ -55,16 +65,16 @@ public class LoadBalancerPerformanceEvaluation extends AbstractHBaseTool { LoggerFactory.getLogger(LoadBalancerPerformanceEvaluation.class.getName()); private static final int DEFAULT_NUM_REGIONS = 1000000; - private static Option NUM_REGIONS_OPT = new Option("regions", true, + private static final Option NUM_REGIONS_OPT = new Option("regions", true, "Number of regions to consider by load balancer. Default: " + DEFAULT_NUM_REGIONS); private static final int DEFAULT_NUM_SERVERS = 1000; - private static Option NUM_SERVERS_OPT = new Option("servers", true, + private static final Option NUM_SERVERS_OPT = new Option("servers", true, "Number of servers to consider by load balancer. Default: " + DEFAULT_NUM_SERVERS); private static final String DEFAULT_LOAD_BALANCER = "org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer"; - private static Option LOAD_BALANCER_OPT = new Option("load_balancer", true, + private static final Option LOAD_BALANCER_OPT = new Option("load_balancer", true, "Type of Load Balancer to use. Default: " + DEFAULT_LOAD_BALANCER); private int numRegions; @@ -85,6 +95,13 @@ public class LoadBalancerPerformanceEvaluation extends AbstractHBaseTool { conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, loadBalancerClazz, LoadBalancer.class); loadBalancer = LoadBalancerFactory.getLoadBalancer(conf); + loadBalancer.setClusterInfoProvider(new DummyClusterInfoProvider(conf)); + try { + loadBalancer.initialize(); + } catch (IOException e) { + LOG.error("Failed to initialize load balancer", e); + throw new RuntimeException("Failed to initialize load balancer", e); + } } private void generateRegionsAndServers() { @@ -152,19 +169,19 @@ public class LoadBalancerPerformanceEvaluation extends AbstractHBaseTool { generateRegionsAndServers(); String methodName = "roundRobinAssignment"; - LOG.info("Calling " + methodName); + LOG.info("Calling {}", methodName); Stopwatch watch = Stopwatch.createStarted(); loadBalancer.roundRobinAssignment(regions, servers); System.out.print(formatResults(methodName, watch.elapsed(TimeUnit.MILLISECONDS))); methodName = "retainAssignment"; - LOG.info("Calling " + methodName); + LOG.info("Calling {}", methodName); watch.reset().start(); loadBalancer.retainAssignment(regionServerMap, servers); System.out.print(formatResults(methodName, watch.elapsed(TimeUnit.MILLISECONDS))); methodName = "balanceCluster"; - LOG.info("Calling " + methodName); + LOG.info("Calling {}", methodName); watch.reset().start(); loadBalancer.balanceCluster(tableServerRegionMap); @@ -178,4 +195,96 @@ public class LoadBalancerPerformanceEvaluation extends AbstractHBaseTool { tool.setConf(HBaseConfiguration.create()); tool.run(args); } + + private static class DummyClusterInfoProvider implements ClusterInfoProvider { + + private volatile Configuration conf; + + public DummyClusterInfoProvider(Configuration conf) { + this.conf = conf; + } + + @Override + public Configuration getConfiguration() { + return conf; + } + + @Override + public Connection getConnection() { + return null; + } + + @Override + public List<RegionInfo> getAssignedRegions() { + return Collections.emptyList(); + } + + @Override + public void unassign(RegionInfo regionInfo) throws IOException { + + } + + @Override + public TableDescriptor getTableDescriptor(TableName tableName) throws IOException { + return null; + } + + @Override + public int getNumberOfTables() throws IOException { + return 0; + } + + @Override + public HDFSBlocksDistribution computeHDFSBlocksDistribution(Configuration conf, + TableDescriptor tableDescriptor, RegionInfo regionInfo) throws IOException { + return new HDFSBlocksDistribution(); + } + + @Override + public boolean hasRegionReplica(Collection<RegionInfo> regions) throws IOException { + return false; + } + + @Override + public List<ServerName> getOnlineServersList() { + return Collections.emptyList(); + } + + @Override + public List<ServerName> getOnlineServersListWithPredicator(List<ServerName> servers, + Predicate<ServerMetrics> filter) { + return Collections.emptyList(); + } + + @Override + public Map<ServerName, List<RegionInfo>> + getSnapShotOfAssignment(Collection<RegionInfo> regions) { + return Collections.emptyMap(); + } + + @Override + public boolean isOffPeakHour() { + return false; + } + + @Override + public void recordBalancerDecision(Supplier<BalancerDecision> decision) { + + } + + @Override + public void recordBalancerRejection(Supplier<BalancerRejection> rejection) { + + } + + @Override + public ServerMetrics getLoad(ServerName serverName) { + return null; + } + + @Override + public void onConfigurationChange(Configuration conf) { + this.conf = conf; + } + } } diff --git a/hbase-diagnostics/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLoadBalancerPerformanceEvaluation.java b/hbase-diagnostics/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLoadBalancerPerformanceEvaluation.java new file mode 100644 index 00000000000..044eb410433 --- /dev/null +++ b/hbase-diagnostics/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLoadBalancerPerformanceEvaluation.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.io.IOException; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.util.AbstractHBaseTool; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag(MasterTests.TAG) +@Tag(LargeTests.TAG) +public class TestLoadBalancerPerformanceEvaluation { + + private static final LoadBalancerPerformanceEvaluation tool = + new LoadBalancerPerformanceEvaluation(); + + @BeforeEach + public void setUpBeforeEach() { + tool.setConf(HBaseConfiguration.create()); + } + + @AfterEach + public void tearDownAfterEach() { + DefaultMetricsSystem.shutdown(); + } + + @Test + public void testLoadBalancerWithDefaultParams() throws IOException { + int ret = tool.run(new String[0]); + assertEquals(AbstractHBaseTool.EXIT_SUCCESS, ret); + } + + @Test + public void testStochasticLoadBalancer() throws Exception { + testLoadBalancer(StochasticLoadBalancer.class); + } + + @Test + public void testSimpleLoadBalancer() throws Exception { + testLoadBalancer(SimpleLoadBalancer.class); + } + + @Test + public void testCacheAwareLoadBalancer() throws Exception { + testLoadBalancer(CacheAwareLoadBalancer.class); + } + + private void testLoadBalancer(Class<? extends LoadBalancer> loadBalancerClass) throws Exception { + String[] args = + { "-regions", "1000", "-servers", "100", "-load_balancer", loadBalancerClass.getName() }; + int ret = tool.run(args); + assertEquals(AbstractHBaseTool.EXIT_SUCCESS, ret); + } + + @Test + public void testInvalidRegions() { + String[] args = { "-regions", "-100" }; + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> tool.run(args)); + assertEquals("Invalid number of regions!", exception.getMessage()); + } + + @Test + public void testInvalidServers() { + String[] args = { "-servers", "0" }; + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> tool.run(args)); + assertEquals("Invalid number of servers!", exception.getMessage()); + } + + @Test + public void testEmptyLoadBalancer() { + String[] args = { "-load_balancer", "" }; + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> tool.run(args)); + assertEquals("Invalid load balancer type!", exception.getMessage()); + } +}
