This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new a444fc509c3 HBASE-28913 LoadBalancerPerformanceEvaluation fails with
NPE (#7745)
a444fc509c3 is described below
commit a444fc509c3dfcbfddabba4236e82815512ecfab
Author: Liu Xiao <[email protected]>
AuthorDate: Tue Feb 24 22:16:54 2026 +0800
HBASE-28913 LoadBalancerPerformanceEvaluation fails with NPE (#7745)
Signed-off-by: Duo Zhang <[email protected]>
---
.../master/balancer/StochasticLoadBalancer.java | 8 +-
.../LoadBalancerPerformanceEvaluation.java | 121 ++++++++++++++++++++-
.../TestLoadBalancerPerformanceEvaluation.java | 103 ++++++++++++++++++
3 files changed, 222 insertions(+), 10 deletions(-)
diff --git
a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
index 075d970fc69..0eedc8fc5c5 100644
---
a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
+++
b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
@@ -291,10 +291,10 @@ public class StochasticLoadBalancer extends
BaseLoadBalancer {
curFunctionCosts = new double[costFunctions.size()];
tempFunctionCosts = new double[costFunctions.size()];
- LOG.info("Loaded config; maxSteps=" + maxSteps + ", runMaxSteps=" +
runMaxSteps
- + ", stepsPerRegion=" + stepsPerRegion + ", maxRunningTime=" +
maxRunningTime + ", isByTable="
- + isByTable + ", CostFunctions=" +
Arrays.toString(getCostFunctionNames())
- + " , sum of multiplier of cost functions = " + sumMultiplier + " etc.");
+ LOG.info(
+ "Loaded config: maxSteps={}, runMaxSteps={}, stepsPerRegion={},
maxRunningTime={}, isByTable={}, CostFunctions={}, sum of multiplier of cost
functions = {} etc.",
+ maxSteps, runMaxSteps, stepsPerRegion, maxRunningTime, isByTable,
+ Arrays.toString(getCostFunctionNames()), sumMultiplier);
}
@Override
diff --git
a/hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerPerformanceEvaluation.java
b/hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerPerformanceEvaluation.java
index 0e3977dc31a..5c8adeedf46 100644
---
a/hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerPerformanceEvaluation.java
+++
b/hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerPerformanceEvaluation.java
@@ -19,18 +19,28 @@ package org.apache.hadoop.hbase.master.balancer;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HDFSBlocksDistribution;
+import org.apache.hadoop.hbase.ServerMetrics;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.BalancerDecision;
+import org.apache.hadoop.hbase.client.BalancerRejection;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.master.LoadBalancer;
import org.apache.hadoop.hbase.util.AbstractHBaseTool;
import org.apache.hadoop.hbase.util.Bytes;
@@ -55,16 +65,16 @@ public class LoadBalancerPerformanceEvaluation extends
AbstractHBaseTool {
LoggerFactory.getLogger(LoadBalancerPerformanceEvaluation.class.getName());
private static final int DEFAULT_NUM_REGIONS = 1000000;
- private static Option NUM_REGIONS_OPT = new Option("regions", true,
+ private static final Option NUM_REGIONS_OPT = new Option("regions", true,
"Number of regions to consider by load balancer. Default: " +
DEFAULT_NUM_REGIONS);
private static final int DEFAULT_NUM_SERVERS = 1000;
- private static Option NUM_SERVERS_OPT = new Option("servers", true,
+ private static final Option NUM_SERVERS_OPT = new Option("servers", true,
"Number of servers to consider by load balancer. Default: " +
DEFAULT_NUM_SERVERS);
private static final String DEFAULT_LOAD_BALANCER =
"org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer";
- private static Option LOAD_BALANCER_OPT = new Option("load_balancer", true,
+ private static final Option LOAD_BALANCER_OPT = new Option("load_balancer",
true,
"Type of Load Balancer to use. Default: " + DEFAULT_LOAD_BALANCER);
private int numRegions;
@@ -85,6 +95,13 @@ public class LoadBalancerPerformanceEvaluation extends
AbstractHBaseTool {
conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
loadBalancerClazz,
LoadBalancer.class);
loadBalancer = LoadBalancerFactory.getLoadBalancer(conf);
+ loadBalancer.setClusterInfoProvider(new DummyClusterInfoProvider(conf));
+ try {
+ loadBalancer.initialize();
+ } catch (IOException e) {
+ LOG.error("Failed to initialize load balancer", e);
+ throw new RuntimeException("Failed to initialize load balancer", e);
+ }
}
private void generateRegionsAndServers() {
@@ -152,19 +169,19 @@ public class LoadBalancerPerformanceEvaluation extends
AbstractHBaseTool {
generateRegionsAndServers();
String methodName = "roundRobinAssignment";
- LOG.info("Calling " + methodName);
+ LOG.info("Calling {}", methodName);
Stopwatch watch = Stopwatch.createStarted();
loadBalancer.roundRobinAssignment(regions, servers);
System.out.print(formatResults(methodName,
watch.elapsed(TimeUnit.MILLISECONDS)));
methodName = "retainAssignment";
- LOG.info("Calling " + methodName);
+ LOG.info("Calling {}", methodName);
watch.reset().start();
loadBalancer.retainAssignment(regionServerMap, servers);
System.out.print(formatResults(methodName,
watch.elapsed(TimeUnit.MILLISECONDS)));
methodName = "balanceCluster";
- LOG.info("Calling " + methodName);
+ LOG.info("Calling {}", methodName);
watch.reset().start();
loadBalancer.balanceCluster(tableServerRegionMap);
@@ -178,4 +195,96 @@ public class LoadBalancerPerformanceEvaluation extends
AbstractHBaseTool {
tool.setConf(HBaseConfiguration.create());
tool.run(args);
}
+
+ private static class DummyClusterInfoProvider implements ClusterInfoProvider
{
+
+ private volatile Configuration conf;
+
+ public DummyClusterInfoProvider(Configuration conf) {
+ this.conf = conf;
+ }
+
+ @Override
+ public Configuration getConfiguration() {
+ return conf;
+ }
+
+ @Override
+ public Connection getConnection() {
+ return null;
+ }
+
+ @Override
+ public List<RegionInfo> getAssignedRegions() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public void unassign(RegionInfo regionInfo) throws IOException {
+
+ }
+
+ @Override
+ public TableDescriptor getTableDescriptor(TableName tableName) throws
IOException {
+ return null;
+ }
+
+ @Override
+ public int getNumberOfTables() throws IOException {
+ return 0;
+ }
+
+ @Override
+ public HDFSBlocksDistribution computeHDFSBlocksDistribution(Configuration
conf,
+ TableDescriptor tableDescriptor, RegionInfo regionInfo) throws
IOException {
+ return new HDFSBlocksDistribution();
+ }
+
+ @Override
+ public boolean hasRegionReplica(Collection<RegionInfo> regions) throws
IOException {
+ return false;
+ }
+
+ @Override
+ public List<ServerName> getOnlineServersList() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public List<ServerName>
getOnlineServersListWithPredicator(List<ServerName> servers,
+ Predicate<ServerMetrics> filter) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public Map<ServerName, List<RegionInfo>>
+ getSnapShotOfAssignment(Collection<RegionInfo> regions) {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public boolean isOffPeakHour() {
+ return false;
+ }
+
+ @Override
+ public void recordBalancerDecision(Supplier<BalancerDecision> decision) {
+
+ }
+
+ @Override
+ public void recordBalancerRejection(Supplier<BalancerRejection> rejection)
{
+
+ }
+
+ @Override
+ public ServerMetrics getLoad(ServerName serverName) {
+ return null;
+ }
+
+ @Override
+ public void onConfigurationChange(Configuration conf) {
+ this.conf = conf;
+ }
+ }
}
diff --git
a/hbase-diagnostics/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLoadBalancerPerformanceEvaluation.java
b/hbase-diagnostics/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLoadBalancerPerformanceEvaluation.java
new file mode 100644
index 00000000000..044eb410433
--- /dev/null
+++
b/hbase-diagnostics/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLoadBalancerPerformanceEvaluation.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.balancer;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import java.io.IOException;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.master.LoadBalancer;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.util.AbstractHBaseTool;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+
+@Tag(MasterTests.TAG)
+@Tag(LargeTests.TAG)
+public class TestLoadBalancerPerformanceEvaluation {
+
+ private static final LoadBalancerPerformanceEvaluation tool =
+ new LoadBalancerPerformanceEvaluation();
+
+ @BeforeEach
+ public void setUpBeforeEach() {
+ tool.setConf(HBaseConfiguration.create());
+ }
+
+ @AfterEach
+ public void tearDownAfterEach() {
+ DefaultMetricsSystem.shutdown();
+ }
+
+ @Test
+ public void testLoadBalancerWithDefaultParams() throws IOException {
+ int ret = tool.run(new String[0]);
+ assertEquals(AbstractHBaseTool.EXIT_SUCCESS, ret);
+ }
+
+ @Test
+ public void testStochasticLoadBalancer() throws Exception {
+ testLoadBalancer(StochasticLoadBalancer.class);
+ }
+
+ @Test
+ public void testSimpleLoadBalancer() throws Exception {
+ testLoadBalancer(SimpleLoadBalancer.class);
+ }
+
+ @Test
+ public void testCacheAwareLoadBalancer() throws Exception {
+ testLoadBalancer(CacheAwareLoadBalancer.class);
+ }
+
+ private void testLoadBalancer(Class<? extends LoadBalancer>
loadBalancerClass) throws Exception {
+ String[] args =
+ { "-regions", "1000", "-servers", "100", "-load_balancer",
loadBalancerClass.getName() };
+ int ret = tool.run(args);
+ assertEquals(AbstractHBaseTool.EXIT_SUCCESS, ret);
+ }
+
+ @Test
+ public void testInvalidRegions() {
+ String[] args = { "-regions", "-100" };
+ IllegalArgumentException exception =
+ assertThrows(IllegalArgumentException.class, () -> tool.run(args));
+ assertEquals("Invalid number of regions!", exception.getMessage());
+ }
+
+ @Test
+ public void testInvalidServers() {
+ String[] args = { "-servers", "0" };
+ IllegalArgumentException exception =
+ assertThrows(IllegalArgumentException.class, () -> tool.run(args));
+ assertEquals("Invalid number of servers!", exception.getMessage());
+ }
+
+ @Test
+ public void testEmptyLoadBalancer() {
+ String[] args = { "-load_balancer", "" };
+ IllegalArgumentException exception =
+ assertThrows(IllegalArgumentException.class, () -> tool.run(args));
+ assertEquals("Invalid load balancer type!", exception.getMessage());
+ }
+}
