This is an automated email from the ASF dual-hosted git repository. ndimiduk pushed a commit to branch branch-2.4 in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.4 by this push: new c602eead5fd HBASE-28068 Add hbase.normalizer.merge.merge_request_max_number_of_regions property … c602eead5fd is described below commit c602eead5fde60c2701c1bcc2c68e5a7b7afdd40 Author: Rahul Kumar <rkrahul...@gmail.com> AuthorDate: Mon Sep 25 17:12:30 2023 +0530 HBASE-28068 Add hbase.normalizer.merge.merge_request_max_number_of_regions property … Authored-by: Rahul Kumar <rahul.ku...@rahulku-ltmdgmc.internal.salesforce.com> Signed-off-by: Nick Dimiduk <ndimi...@apache.org> --- hbase-common/src/main/resources/hbase-default.xml | 6 +++ .../master/normalizer/SimpleRegionNormalizer.java | 55 +++++++++++++++++----- .../normalizer/TestSimpleRegionNormalizer.java | 34 +++++++++++++ 3 files changed, 82 insertions(+), 13 deletions(-) diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index c84eb8d183d..77a44694c29 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -649,6 +649,12 @@ possible configurations would overwhelm and obscure the important. <description>The minimum size for a region to be considered for a merge, in whole MBs.</description> </property> + <property> + <name>hbase.normalizer.merge.merge_request_max_number_of_regions</name> + <value>50</value> + <description>The maximum number of region count in a merge request for merge + normalization.</description> + </property> <property> <name>hbase.table.normalization.enabled</name> <value>false</value> diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java index 9efe96ed766..12bd37f7ff8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java @@ -74,6 +74,9 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver static final int DEFAULT_MERGE_MIN_REGION_AGE_DAYS = 3; static final String MERGE_MIN_REGION_SIZE_MB_KEY = "hbase.normalizer.merge.min_region_size.mb"; static final int DEFAULT_MERGE_MIN_REGION_SIZE_MB = 1; + static final String MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY = + "hbase.normalizer.merge.merge_request_max_number_of_regions"; + static final long DEFAULT_MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT = 100; private MasterServices masterServices; private NormalizerConfiguration normalizerConfiguration; @@ -131,6 +134,16 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver return settledValue; } + private static long parseMergeRequestMaxNumberOfRegionsCount(final Configuration conf) { + final long parsedValue = conf.getLong(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY, + DEFAULT_MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT); + final long settledValue = Math.max(2, parsedValue); + if (parsedValue != settledValue) { + warnInvalidValue(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY, parsedValue, settledValue); + } + return settledValue; + } + private static <T> void warnInvalidValue(final String key, final T parsedValue, final T settledValue) { LOG.warn("Configured value {}={} is invalid. Setting value to {}.", key, parsedValue, @@ -179,6 +192,10 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver return normalizerConfiguration.getMergeMinRegionSizeMb(); } + public long getMergeRequestMaxNumberOfRegionsCount() { + return normalizerConfiguration.getMergeRequestMaxNumberOfRegionsCount(); + } + @Override public void setMasterServices(final MasterServices masterServices) { this.masterServices = masterServices; @@ -367,19 +384,21 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver break; } if ( - rangeMembers.isEmpty() // when there are no range members, seed the range with whatever - // we have. this way we're prepared in case the next region is - // 0-size. - || (rangeMembers.size() == 1 && sumRangeMembersSizeMb == 0) // when there is only one - // region and the size is 0, - // seed the range with - // whatever we have. - || regionSizeMb == 0 // always add an empty region to the current range. - || (regionSizeMb + sumRangeMembersSizeMb <= avgRegionSizeMb) - ) { // add the current region - // to the range when - // there's capacity - // remaining. + // when there are no range members, seed the range with whatever we have. this way we're + // prepared in case the next region is 0-size. + rangeMembers.isEmpty() + // when there is only one region and the size is 0, seed the range with whatever we + // have. + || (rangeMembers.size() == 1 && sumRangeMembersSizeMb == 0) + // add an empty region to the current range only if it doesn't exceed max merge request + // region count + || (regionSizeMb == 0 && rangeMembers.size() < getMergeRequestMaxNumberOfRegionsCount()) + // add region if current range region size is less than avg region size of table + // and current range doesn't exceed max merge request region count + || ((regionSizeMb + sumRangeMembersSizeMb <= avgRegionSizeMb) + && (rangeMembers.size() < getMergeRequestMaxNumberOfRegionsCount())) + ) { + // add the current region to the range when there's capacity remaining. rangeMembers.add(new NormalizationTarget(regionInfo, regionSizeMb)); sumRangeMembersSizeMb += regionSizeMb; continue; @@ -479,6 +498,7 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver private final int minRegionCount; private final Period mergeMinRegionAge; private final long mergeMinRegionSizeMb; + private final long mergeRequestMaxNumberOfRegionsCount; private NormalizerConfiguration() { conf = null; @@ -487,6 +507,7 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver minRegionCount = DEFAULT_MIN_REGION_COUNT; mergeMinRegionAge = Period.ofDays(DEFAULT_MERGE_MIN_REGION_AGE_DAYS); mergeMinRegionSizeMb = DEFAULT_MERGE_MIN_REGION_SIZE_MB; + mergeRequestMaxNumberOfRegionsCount = DEFAULT_MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT; } private NormalizerConfiguration(final Configuration conf, @@ -497,6 +518,7 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver minRegionCount = parseMinRegionCount(conf); mergeMinRegionAge = parseMergeMinRegionAge(conf); mergeMinRegionSizeMb = parseMergeMinRegionSizeMb(conf); + mergeRequestMaxNumberOfRegionsCount = parseMergeRequestMaxNumberOfRegionsCount(conf); logConfigurationUpdated(SPLIT_ENABLED_KEY, currentConfiguration.isSplitEnabled(), splitEnabled); logConfigurationUpdated(MERGE_ENABLED_KEY, currentConfiguration.isMergeEnabled(), @@ -507,6 +529,9 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver currentConfiguration.getMergeMinRegionAge(), mergeMinRegionAge); logConfigurationUpdated(MERGE_MIN_REGION_SIZE_MB_KEY, currentConfiguration.getMergeMinRegionSizeMb(), mergeMinRegionSizeMb); + logConfigurationUpdated(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY, + currentConfiguration.getMergeRequestMaxNumberOfRegionsCount(), + mergeRequestMaxNumberOfRegionsCount); } public Configuration getConf() { @@ -558,6 +583,10 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver } return mergeMinRegionSizeMb; } + + public long getMergeRequestMaxNumberOfRegionsCount() { + return mergeRequestMaxNumberOfRegionsCount; + } } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java index da3e8db430f..3dd4d5404a7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java @@ -22,6 +22,7 @@ import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.D import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_ENABLED_KEY; import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_AGE_DAYS_KEY; import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_SIZE_MB_KEY; +import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY; import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MIN_REGION_COUNT_KEY; import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.SPLIT_ENABLED_KEY; import static org.hamcrest.MatcherAssert.assertThat; @@ -468,6 +469,39 @@ public class TestSimpleRegionNormalizer { assertThat(normalizer.computePlansForTable(tableDescriptor), empty()); } + @Test + public void testHonorsMergeRequestMaxNumberOfRegionsCount() { + conf.setBoolean(SPLIT_ENABLED_KEY, false); + conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0); + conf.setInt(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY, 3); + final TableName tableName = name.getTableName(); + final List<RegionInfo> regionInfos = createRegionInfos(tableName, 5); + final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 0, 1, 0, 1, 0); + setupMocksForNormalizer(regionSizes, regionInfos); + assertEquals(3, normalizer.getMergeRequestMaxNumberOfRegionsCount()); + List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor); + assertThat(plans, + contains( + new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0) + .addTarget(regionInfos.get(1), 1).addTarget(regionInfos.get(2), 0).build(), + new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(3), 1) + .addTarget(regionInfos.get(4), 0).build())); + } + + @Test + public void testHonorsMergeRequestMaxNumberOfRegionsCountDefault() { + conf.setBoolean(SPLIT_ENABLED_KEY, false); + conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0); + final TableName tableName = name.getTableName(); + final List<RegionInfo> regionInfos = createRegionInfos(tableName, 3); + final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 0, 0, 0); + setupMocksForNormalizer(regionSizes, regionInfos); + assertEquals(50, normalizer.getMergeRequestMaxNumberOfRegionsCount()); + List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor); + assertThat(plans, contains(new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0) + .addTarget(regionInfos.get(1), 0).addTarget(regionInfos.get(2), 0).build())); + } + @Test public void testMergeEmptyRegions0() { conf.setBoolean(SPLIT_ENABLED_KEY, false);