This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new c602eead5fd HBASE-28068 Add
hbase.normalizer.merge.merge_request_max_number_of_regions property …
c602eead5fd is described below
commit c602eead5fde60c2701c1bcc2c68e5a7b7afdd40
Author: Rahul Kumar <[email protected]>
AuthorDate: Mon Sep 25 17:12:30 2023 +0530
HBASE-28068 Add hbase.normalizer.merge.merge_request_max_number_of_regions
property …
Authored-by: Rahul Kumar
<[email protected]>
Signed-off-by: Nick Dimiduk <[email protected]>
---
hbase-common/src/main/resources/hbase-default.xml | 6 +++
.../master/normalizer/SimpleRegionNormalizer.java | 55 +++++++++++++++++-----
.../normalizer/TestSimpleRegionNormalizer.java | 34 +++++++++++++
3 files changed, 82 insertions(+), 13 deletions(-)
diff --git a/hbase-common/src/main/resources/hbase-default.xml
b/hbase-common/src/main/resources/hbase-default.xml
index c84eb8d183d..77a44694c29 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -649,6 +649,12 @@ possible configurations would overwhelm and obscure the
important.
<description>The minimum size for a region to be considered for a merge,
in whole
MBs.</description>
</property>
+ <property>
+ <name>hbase.normalizer.merge.merge_request_max_number_of_regions</name>
+ <value>50</value>
+ <description>The maximum number of region count in a merge request for
merge
+ normalization.</description>
+ </property>
<property>
<name>hbase.table.normalization.enabled</name>
<value>false</value>
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
index 9efe96ed766..12bd37f7ff8 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
@@ -74,6 +74,9 @@ class SimpleRegionNormalizer implements RegionNormalizer,
ConfigurationObserver
static final int DEFAULT_MERGE_MIN_REGION_AGE_DAYS = 3;
static final String MERGE_MIN_REGION_SIZE_MB_KEY =
"hbase.normalizer.merge.min_region_size.mb";
static final int DEFAULT_MERGE_MIN_REGION_SIZE_MB = 1;
+ static final String MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY =
+ "hbase.normalizer.merge.merge_request_max_number_of_regions";
+ static final long DEFAULT_MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT = 100;
private MasterServices masterServices;
private NormalizerConfiguration normalizerConfiguration;
@@ -131,6 +134,16 @@ class SimpleRegionNormalizer implements RegionNormalizer,
ConfigurationObserver
return settledValue;
}
+ private static long parseMergeRequestMaxNumberOfRegionsCount(final
Configuration conf) {
+ final long parsedValue =
conf.getLong(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY,
+ DEFAULT_MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT);
+ final long settledValue = Math.max(2, parsedValue);
+ if (parsedValue != settledValue) {
+ warnInvalidValue(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY,
parsedValue, settledValue);
+ }
+ return settledValue;
+ }
+
private static <T> void warnInvalidValue(final String key, final T
parsedValue,
final T settledValue) {
LOG.warn("Configured value {}={} is invalid. Setting value to {}.", key,
parsedValue,
@@ -179,6 +192,10 @@ class SimpleRegionNormalizer implements RegionNormalizer,
ConfigurationObserver
return normalizerConfiguration.getMergeMinRegionSizeMb();
}
+ public long getMergeRequestMaxNumberOfRegionsCount() {
+ return normalizerConfiguration.getMergeRequestMaxNumberOfRegionsCount();
+ }
+
@Override
public void setMasterServices(final MasterServices masterServices) {
this.masterServices = masterServices;
@@ -367,19 +384,21 @@ class SimpleRegionNormalizer implements RegionNormalizer,
ConfigurationObserver
break;
}
if (
- rangeMembers.isEmpty() // when there are no range members, seed the
range with whatever
- // we have. this way we're prepared in case
the next region is
- // 0-size.
- || (rangeMembers.size() == 1 && sumRangeMembersSizeMb == 0) //
when there is only one
- //
region and the size is 0,
- //
seed the range with
- //
whatever we have.
- || regionSizeMb == 0 // always add an empty region to the current
range.
- || (regionSizeMb + sumRangeMembersSizeMb <= avgRegionSizeMb)
- ) { // add the current region
- // to the range when
- // there's capacity
- // remaining.
+ // when there are no range members, seed the range with whatever we
have. this way we're
+ // prepared in case the next region is 0-size.
+ rangeMembers.isEmpty()
+ // when there is only one region and the size is 0, seed the range
with whatever we
+ // have.
+ || (rangeMembers.size() == 1 && sumRangeMembersSizeMb == 0)
+ // add an empty region to the current range only if it doesn't
exceed max merge request
+ // region count
+ || (regionSizeMb == 0 && rangeMembers.size() <
getMergeRequestMaxNumberOfRegionsCount())
+ // add region if current range region size is less than avg region
size of table
+ // and current range doesn't exceed max merge request region count
+ || ((regionSizeMb + sumRangeMembersSizeMb <= avgRegionSizeMb)
+ && (rangeMembers.size() <
getMergeRequestMaxNumberOfRegionsCount()))
+ ) {
+ // add the current region to the range when there's capacity
remaining.
rangeMembers.add(new NormalizationTarget(regionInfo, regionSizeMb));
sumRangeMembersSizeMb += regionSizeMb;
continue;
@@ -479,6 +498,7 @@ class SimpleRegionNormalizer implements RegionNormalizer,
ConfigurationObserver
private final int minRegionCount;
private final Period mergeMinRegionAge;
private final long mergeMinRegionSizeMb;
+ private final long mergeRequestMaxNumberOfRegionsCount;
private NormalizerConfiguration() {
conf = null;
@@ -487,6 +507,7 @@ class SimpleRegionNormalizer implements RegionNormalizer,
ConfigurationObserver
minRegionCount = DEFAULT_MIN_REGION_COUNT;
mergeMinRegionAge = Period.ofDays(DEFAULT_MERGE_MIN_REGION_AGE_DAYS);
mergeMinRegionSizeMb = DEFAULT_MERGE_MIN_REGION_SIZE_MB;
+ mergeRequestMaxNumberOfRegionsCount =
DEFAULT_MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT;
}
private NormalizerConfiguration(final Configuration conf,
@@ -497,6 +518,7 @@ class SimpleRegionNormalizer implements RegionNormalizer,
ConfigurationObserver
minRegionCount = parseMinRegionCount(conf);
mergeMinRegionAge = parseMergeMinRegionAge(conf);
mergeMinRegionSizeMb = parseMergeMinRegionSizeMb(conf);
+ mergeRequestMaxNumberOfRegionsCount =
parseMergeRequestMaxNumberOfRegionsCount(conf);
logConfigurationUpdated(SPLIT_ENABLED_KEY,
currentConfiguration.isSplitEnabled(),
splitEnabled);
logConfigurationUpdated(MERGE_ENABLED_KEY,
currentConfiguration.isMergeEnabled(),
@@ -507,6 +529,9 @@ class SimpleRegionNormalizer implements RegionNormalizer,
ConfigurationObserver
currentConfiguration.getMergeMinRegionAge(), mergeMinRegionAge);
logConfigurationUpdated(MERGE_MIN_REGION_SIZE_MB_KEY,
currentConfiguration.getMergeMinRegionSizeMb(), mergeMinRegionSizeMb);
+ logConfigurationUpdated(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY,
+ currentConfiguration.getMergeRequestMaxNumberOfRegionsCount(),
+ mergeRequestMaxNumberOfRegionsCount);
}
public Configuration getConf() {
@@ -558,6 +583,10 @@ class SimpleRegionNormalizer implements RegionNormalizer,
ConfigurationObserver
}
return mergeMinRegionSizeMb;
}
+
+ public long getMergeRequestMaxNumberOfRegionsCount() {
+ return mergeRequestMaxNumberOfRegionsCount;
+ }
}
/**
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
index da3e8db430f..3dd4d5404a7 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
@@ -22,6 +22,7 @@ import static
org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.D
import static
org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_ENABLED_KEY;
import static
org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_AGE_DAYS_KEY;
import static
org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_SIZE_MB_KEY;
+import static
org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY;
import static
org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MIN_REGION_COUNT_KEY;
import static
org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.SPLIT_ENABLED_KEY;
import static org.hamcrest.MatcherAssert.assertThat;
@@ -468,6 +469,39 @@ public class TestSimpleRegionNormalizer {
assertThat(normalizer.computePlansForTable(tableDescriptor), empty());
}
+ @Test
+ public void testHonorsMergeRequestMaxNumberOfRegionsCount() {
+ conf.setBoolean(SPLIT_ENABLED_KEY, false);
+ conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0);
+ conf.setInt(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY, 3);
+ final TableName tableName = name.getTableName();
+ final List<RegionInfo> regionInfos = createRegionInfos(tableName, 5);
+ final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos,
0, 1, 0, 1, 0);
+ setupMocksForNormalizer(regionSizes, regionInfos);
+ assertEquals(3, normalizer.getMergeRequestMaxNumberOfRegionsCount());
+ List<NormalizationPlan> plans =
normalizer.computePlansForTable(tableDescriptor);
+ assertThat(plans,
+ contains(
+ new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0)
+ .addTarget(regionInfos.get(1), 1).addTarget(regionInfos.get(2),
0).build(),
+ new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(3), 1)
+ .addTarget(regionInfos.get(4), 0).build()));
+ }
+
+ @Test
+ public void testHonorsMergeRequestMaxNumberOfRegionsCountDefault() {
+ conf.setBoolean(SPLIT_ENABLED_KEY, false);
+ conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0);
+ final TableName tableName = name.getTableName();
+ final List<RegionInfo> regionInfos = createRegionInfos(tableName, 3);
+ final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos,
0, 0, 0);
+ setupMocksForNormalizer(regionSizes, regionInfos);
+ assertEquals(50, normalizer.getMergeRequestMaxNumberOfRegionsCount());
+ List<NormalizationPlan> plans =
normalizer.computePlansForTable(tableDescriptor);
+ assertThat(plans, contains(new
MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0)
+ .addTarget(regionInfos.get(1), 0).addTarget(regionInfos.get(2),
0).build()));
+ }
+
@Test
public void testMergeEmptyRegions0() {
conf.setBoolean(SPLIT_ENABLED_KEY, false);
