This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 17652a7 HBASE-23590 : Update maxStoreFileRefCount to
maxCompactedStoreFileRefCount for auto region recovery based on old reader
references
17652a7 is described below
commit 17652a7b323ea4266c047843ca3c5fed613a0d4a
Author: Viraj Jasani <[email protected]>
AuthorDate: Wed Jan 1 21:57:03 2020 +0530
HBASE-23590 : Update maxStoreFileRefCount to maxCompactedStoreFileRefCount
for auto region recovery based on old reader references
Signed-off-by: Anoop Sam John <[email protected]>
---
.../java/org/apache/hadoop/hbase/RegionLoad.java | 4 +--
.../org/apache/hadoop/hbase/RegionMetrics.java | 4 +--
.../apache/hadoop/hbase/RegionMetricsBuilder.java | 26 +++++++++---------
.../apache/hadoop/hbase/ServerMetricsBuilder.java | 10 ++++---
hbase-common/src/main/resources/hbase-default.xml | 32 +++++++++++++---------
.../regionserver/MetricsRegionServerSource.java | 2 +-
.../hbase/regionserver/MetricsRegionWrapper.java | 4 +--
.../regionserver/MetricsRegionSourceImpl.java | 7 +++--
.../regionserver/TestMetricsRegionSourceImpl.java | 2 +-
.../src/main/protobuf/ClusterStatus.proto | 4 +--
.../src/main/protobuf/ClusterStatus.proto | 4 +--
.../hadoop/hbase/master/RegionsRecoveryChore.java | 19 +++++++------
.../hadoop/hbase/regionserver/HRegionServer.java | 9 +++---
.../apache/hadoop/hbase/regionserver/HStore.java | 11 ++++----
.../regionserver/MetricsRegionWrapperImpl.java | 16 +++++------
.../hbase/master/TestRegionsRecoveryChore.java | 12 ++++----
.../regionserver/MetricsRegionWrapperStub.java | 2 +-
src/main/asciidoc/_chapters/hbase-default.adoc | 32 +++++++++++++---------
18 files changed, 109 insertions(+), 91 deletions(-)
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
index b48bbb8..fcaec60 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
@@ -366,8 +366,8 @@ public class RegionLoad implements RegionMetrics {
}
@Override
- public int getMaxStoreFileRefCount() {
- return metrics.getMaxStoreFileRefCount();
+ public int getMaxCompactedStoreFileRefCount() {
+ return metrics.getMaxCompactedStoreFileRefCount();
}
/**
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java
index 8ce61c7..eab57dc 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java
@@ -150,8 +150,8 @@ public interface RegionMetrics {
int getStoreRefCount();
/**
- * @return the max reference count for any store file among all stores files
+ * @return the max reference count for any store file among all compacted
stores files
* of this region
*/
- int getMaxStoreFileRefCount();
+ int getMaxCompactedStoreFileRefCount();
}
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java
index a43350d..da0f4bf 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java
@@ -65,7 +65,7 @@ public final class RegionMetricsBuilder {
.setStoreCount(regionLoadPB.getStores())
.setStoreFileCount(regionLoadPB.getStorefiles())
.setStoreRefCount(regionLoadPB.getStoreRefCount())
- .setMaxStoreFileRefCount(regionLoadPB.getMaxStoreFileRefCount())
+
.setMaxCompactedStoreFileRefCount(regionLoadPB.getMaxCompactedStoreFileRefCount())
.setStoreFileSize(new Size(regionLoadPB.getStorefileSizeMB(),
Size.Unit.MEGABYTE))
.setStoreSequenceIds(regionLoadPB.getStoreCompleteSequenceIdList().stream()
.collect(Collectors.toMap(
@@ -112,7 +112,7 @@ public final class RegionMetricsBuilder {
.setStores(regionMetrics.getStoreCount())
.setStorefiles(regionMetrics.getStoreFileCount())
.setStoreRefCount(regionMetrics.getStoreRefCount())
- .setMaxStoreFileRefCount(regionMetrics.getMaxStoreFileRefCount())
+
.setMaxCompactedStoreFileRefCount(regionMetrics.getMaxCompactedStoreFileRefCount())
.setStorefileSizeMB((int)
regionMetrics.getStoreFileSize().get(Size.Unit.MEGABYTE))
.addAllStoreCompleteSequenceId(toStoreSequenceId(regionMetrics.getStoreSequenceId()))
.setStoreUncompressedSizeMB(
@@ -128,7 +128,7 @@ public final class RegionMetricsBuilder {
private int storeCount;
private int storeFileCount;
private int storeRefCount;
- private int maxStoreFileRefCount;
+ private int maxCompactedStoreFileRefCount;
private long compactingCellCount;
private long compactedCellCount;
private Size storeFileSize = Size.ZERO;
@@ -161,8 +161,8 @@ public final class RegionMetricsBuilder {
this.storeRefCount = value;
return this;
}
- public RegionMetricsBuilder setMaxStoreFileRefCount(int value) {
- this.maxStoreFileRefCount = value;
+ public RegionMetricsBuilder setMaxCompactedStoreFileRefCount(int value) {
+ this.maxCompactedStoreFileRefCount = value;
return this;
}
public RegionMetricsBuilder setCompactingCellCount(long value) {
@@ -235,7 +235,7 @@ public final class RegionMetricsBuilder {
storeCount,
storeFileCount,
storeRefCount,
- maxStoreFileRefCount,
+ maxCompactedStoreFileRefCount,
compactingCellCount,
compactedCellCount,
storeFileSize,
@@ -259,7 +259,7 @@ public final class RegionMetricsBuilder {
private final int storeCount;
private final int storeFileCount;
private final int storeRefCount;
- private final int maxStoreFileRefCount;
+ private final int maxCompactedStoreFileRefCount;
private final long compactingCellCount;
private final long compactedCellCount;
private final Size storeFileSize;
@@ -280,7 +280,7 @@ public final class RegionMetricsBuilder {
int storeCount,
int storeFileCount,
int storeRefCount,
- int maxStoreFileRefCount,
+ int maxCompactedStoreFileRefCount,
final long compactingCellCount,
long compactedCellCount,
Size storeFileSize,
@@ -301,7 +301,7 @@ public final class RegionMetricsBuilder {
this.storeCount = storeCount;
this.storeFileCount = storeFileCount;
this.storeRefCount = storeRefCount;
- this.maxStoreFileRefCount = maxStoreFileRefCount;
+ this.maxCompactedStoreFileRefCount = maxCompactedStoreFileRefCount;
this.compactingCellCount = compactingCellCount;
this.compactedCellCount = compactedCellCount;
this.storeFileSize = Preconditions.checkNotNull(storeFileSize);
@@ -341,8 +341,8 @@ public final class RegionMetricsBuilder {
}
@Override
- public int getMaxStoreFileRefCount() {
- return maxStoreFileRefCount;
+ public int getMaxCompactedStoreFileRefCount() {
+ return maxCompactedStoreFileRefCount;
}
@Override
@@ -433,8 +433,8 @@ public final class RegionMetricsBuilder {
this.getStoreFileCount());
Strings.appendKeyValue(sb, "storeRefCount",
this.getStoreRefCount());
- Strings.appendKeyValue(sb, "maxStoreFileRefCount",
- this.getMaxStoreFileRefCount());
+ Strings.appendKeyValue(sb, "maxCompactedStoreFileRefCount",
+ this.getMaxCompactedStoreFileRefCount());
Strings.appendKeyValue(sb, "uncompressedStoreFileSize",
this.getUncompressedStoreFileSize());
Strings.appendKeyValue(sb, "lastMajorCompactionTimestamp",
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetricsBuilder.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetricsBuilder.java
index e5cd7b2..ddb279c 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetricsBuilder.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetricsBuilder.java
@@ -360,7 +360,7 @@ public final class ServerMetricsBuilder {
int storeCount = 0;
int storeFileCount = 0;
int storeRefCount = 0;
- int maxStoreFileRefCount = 0;
+ int maxCompactedStoreFileRefCount = 0;
long uncompressedStoreFileSizeMB = 0;
long storeFileSizeMB = 0;
long memStoreSizeMB = 0;
@@ -376,8 +376,9 @@ public final class ServerMetricsBuilder {
storeCount += r.getStoreCount();
storeFileCount += r.getStoreFileCount();
storeRefCount += r.getStoreRefCount();
- int currentMaxStoreFileRefCount = r.getMaxStoreFileRefCount();
- maxStoreFileRefCount = Math.max(maxStoreFileRefCount,
currentMaxStoreFileRefCount);
+ int currentMaxCompactedStoreFileRefCount =
r.getMaxCompactedStoreFileRefCount();
+ maxCompactedStoreFileRefCount = Math.max(maxCompactedStoreFileRefCount,
+ currentMaxCompactedStoreFileRefCount);
uncompressedStoreFileSizeMB +=
r.getUncompressedStoreFileSize().get(Size.Unit.MEGABYTE);
storeFileSizeMB += r.getStoreFileSize().get(Size.Unit.MEGABYTE);
memStoreSizeMB += r.getMemStoreSize().get(Size.Unit.MEGABYTE);
@@ -399,7 +400,8 @@ public final class ServerMetricsBuilder {
Strings.appendKeyValue(sb, "numberOfStores", storeCount);
Strings.appendKeyValue(sb, "numberOfStorefiles", storeFileCount);
Strings.appendKeyValue(sb, "storeRefCount", storeRefCount);
- Strings.appendKeyValue(sb, "maxStoreFileRefCount", maxStoreFileRefCount);
+ Strings.appendKeyValue(sb, "maxCompactedStoreFileRefCount",
+ maxCompactedStoreFileRefCount);
Strings.appendKeyValue(sb, "storefileUncompressedSizeMB",
uncompressedStoreFileSizeMB);
Strings.appendKeyValue(sb, "storefileSizeMB", storeFileSizeMB);
if (uncompressedStoreFileSizeMB != 0) {
diff --git a/hbase-common/src/main/resources/hbase-default.xml
b/hbase-common/src/main/resources/hbase-default.xml
index cc1b728..7b07793 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -1891,19 +1891,25 @@ possible configurations would overwhelm and obscure the
important.
<name>hbase.regions.recovery.store.file.ref.count</name>
<value>-1</value>
<description>
- Very large ref count on a file indicates
- that it is a ref leak on that object. Such files
- can not be removed even after it is invalidated
- via compaction. Only way to recover in such
- scenario is to reopen the region which can
- release all resources, like the refcount, leases, etc.
- This config represents Store files Ref Count threshold
- value considered for reopening regions.
- Any region with store files ref count > this value
- would be eligible for reopening by master.
- Default value -1 indicates this feature is turned off.
- Only positive integer value should be provided to enable
- this feature.
+ Very large number of ref count on a compacted
+ store file indicates that it is a ref leak
+ on that object(compacted store file).
+ Such files can not be removed after
+ it is invalidated via compaction.
+ Only way to recover in such scenario is to
+ reopen the region which can release
+ all resources, like the refcount,
+ leases, etc. This config represents Store files Ref
+ Count threshold value considered for reopening
+ regions. Any region with compacted store files
+ ref count > this value would be eligible for
+ reopening by master. Here, we get the max
+ refCount among all refCounts on all
+ compacted away store files that belong to a
+ particular region. Default value -1 indicates
+ this feature is turned off. Only positive
+ integer value should be provided to
+ enable this feature.
</description>
</property>
</configuration>
diff --git
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
index 5923b13..8f8a12d 100644
---
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
+++
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
@@ -233,7 +233,7 @@ public interface MetricsRegionServerSource extends
BaseSource, JvmPauseMonitorSo
String STOREFILE_COUNT_DESC = "Number of Store Files";
String STORE_REF_COUNT = "storeRefCount";
String STORE_REF_COUNT_DESC = "Store reference count";
- String MAX_STORE_FILE_REF_COUNT = "maxStoreFileRefCount";
+ String MAX_COMPACTED_STORE_FILE_REF_COUNT = "maxCompactedStoreFileRefCount";
String MEMSTORE_SIZE = "memStoreSize";
String MEMSTORE_SIZE_DESC = "Size of the memstore";
String STOREFILE_SIZE = "storeFileSize";
diff --git
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
index b7212f0..34552ed 100644
---
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
+++
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
@@ -160,7 +160,7 @@ public interface MetricsRegionWrapper {
/**
* @return the max number of references active on any store file among
- * all store files that belong to this region
+ * all compacted store files that belong to this region
*/
- long getMaxStoreFileRefCount();
+ long getMaxCompactedStoreFileRefCount();
}
diff --git
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
index 32e3125..6198717 100644
---
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
+++
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
@@ -218,9 +218,10 @@ public class MetricsRegionSourceImpl implements
MetricsRegionSource {
MetricsRegionServerSource.STORE_REF_COUNT),
this.regionWrapper.getStoreRefCount());
mrb.addGauge(Interns.info(
- regionNamePrefix + MetricsRegionServerSource.MAX_STORE_FILE_REF_COUNT,
- MetricsRegionServerSource.MAX_STORE_FILE_REF_COUNT),
- this.regionWrapper.getMaxStoreFileRefCount());
+ regionNamePrefix +
MetricsRegionServerSource.MAX_COMPACTED_STORE_FILE_REF_COUNT,
+ MetricsRegionServerSource.MAX_COMPACTED_STORE_FILE_REF_COUNT),
+ this.regionWrapper.getMaxCompactedStoreFileRefCount()
+ );
mrb.addGauge(Interns.info(
regionNamePrefix + MetricsRegionServerSource.MEMSTORE_SIZE,
MetricsRegionServerSource.MEMSTORE_SIZE_DESC),
diff --git
a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
index ceb7dbb..837e7da 100644
---
a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
+++
b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
@@ -100,7 +100,7 @@ public class TestMetricsRegionSourceImpl {
}
@Override
- public long getMaxStoreFileRefCount() {
+ public long getMaxCompactedStoreFileRefCount() {
return 0;
}
diff --git a/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto
b/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto
index 77b9887..65c2268 100644
--- a/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto
@@ -151,10 +151,10 @@ message RegionLoad {
optional int32 store_ref_count = 21 [default = 0];
/**
- * The max number of references active on single store file among all store
files
+ * The max number of references active on single store file among all
compacted store files
* that belong to given region
*/
- optional int32 max_store_file_ref_count = 22 [default = 0];
+ optional int32 max_compacted_store_file_ref_count = 22 [default = 0];
}
message UserLoad {
diff --git a/hbase-protocol/src/main/protobuf/ClusterStatus.proto
b/hbase-protocol/src/main/protobuf/ClusterStatus.proto
index 0234738..13ec5b7 100644
--- a/hbase-protocol/src/main/protobuf/ClusterStatus.proto
+++ b/hbase-protocol/src/main/protobuf/ClusterStatus.proto
@@ -147,10 +147,10 @@ message RegionLoad {
optional int32 store_ref_count = 21 [default = 0];
/**
- * The max number of references active on single store file among all store
files
+ * The max number of references active on single store file among all
compacted store files
* that belong to given region
*/
- optional int32 max_store_file_ref_count = 22 [default = 0];
+ optional int32 max_compacted_store_file_ref_count = 22 [default = 0];
}
message UserLoad {
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionsRecoveryChore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionsRecoveryChore.java
index c5ad867..0ee5a1b 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionsRecoveryChore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionsRecoveryChore.java
@@ -130,17 +130,18 @@ public class RegionsRecoveryChore extends ScheduledChore {
for (ServerMetrics serverMetrics : serverMetricsMap.values()) {
Map<byte[], RegionMetrics> regionMetricsMap =
serverMetrics.getRegionMetrics();
for (RegionMetrics regionMetrics : regionMetricsMap.values()) {
- // For each region, each store file can have different ref counts
- // We need to find maximum of all such ref counts and if that max count
- // is beyond a threshold value, we should reopen the region.
- // Here, we take max ref count of all store files and not the
cumulative
- // count of all store files
- final int maxStoreFileRefCount =
regionMetrics.getMaxStoreFileRefCount();
-
- if (maxStoreFileRefCount > storeFileRefCountThreshold) {
+ // For each region, each compacted store file can have different ref
counts
+ // We need to find maximum of all such ref counts and if that max
count of compacted
+ // store files is beyond a threshold value, we should reopen the
region.
+ // Here, we take max ref count of all compacted store files and not
the cumulative
+ // count of all compacted store files
+ final int maxCompactedStoreFileRefCount = regionMetrics
+ .getMaxCompactedStoreFileRefCount();
+
+ if (maxCompactedStoreFileRefCount > storeFileRefCountThreshold) {
final byte[] regionName = regionMetrics.getRegionName();
prepareTableToReopenRegionsMap(tableToReopenRegionsMap, regionName,
- maxStoreFileRefCount);
+ maxCompactedStoreFileRefCount);
}
}
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 1c1f0b0..d8a0538 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1660,7 +1660,7 @@ public class HRegionServer extends HasThread implements
int stores = 0;
int storefiles = 0;
int storeRefCount = 0;
- int maxStoreFileRefCount = 0;
+ int maxCompactedStoreFileRefCount = 0;
int storeUncompressedSizeMB = 0;
int storefileSizeMB = 0;
int memstoreSizeMB = (int) (r.getMemStoreDataSize() / 1024 / 1024);
@@ -1676,8 +1676,9 @@ public class HRegionServer extends HasThread implements
storefiles += store.getStorefilesCount();
int currentStoreRefCount = store.getStoreRefCount();
storeRefCount += currentStoreRefCount;
- int currentMaxStoreFileRefCount = store.getMaxStoreFileRefCount();
- maxStoreFileRefCount = Math.max(maxStoreFileRefCount,
currentMaxStoreFileRefCount);
+ int currentMaxCompactedStoreFileRefCount =
store.getMaxCompactedStoreFileRefCount();
+ maxCompactedStoreFileRefCount = Math.max(maxCompactedStoreFileRefCount,
+ currentMaxCompactedStoreFileRefCount);
storeUncompressedSizeMB += (int) (store.getStoreSizeUncompressed() /
1024 / 1024);
storefileSizeMB += (int) (store.getStorefilesSize() / 1024 / 1024);
//TODO: storefileIndexSizeKB is same with rootLevelIndexSizeKB?
@@ -1706,7 +1707,7 @@ public class HRegionServer extends HasThread implements
.setStores(stores)
.setStorefiles(storefiles)
.setStoreRefCount(storeRefCount)
- .setMaxStoreFileRefCount(maxStoreFileRefCount)
+ .setMaxCompactedStoreFileRefCount(maxCompactedStoreFileRefCount)
.setStoreUncompressedSizeMB(storeUncompressedSizeMB)
.setStorefileSizeMB(storefileSizeMB)
.setMemStoreSizeMB(memstoreSizeMB)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index bc0d694..252371d 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -2779,18 +2779,19 @@ public class HStore implements Store, HeapSize,
StoreConfigInformation, Propagat
}
/**
- * @return get maximum ref count of storeFile among all HStore Files
+ * @return get maximum ref count of storeFile among all compacted HStore
Files
* for the HStore
*/
- public int getMaxStoreFileRefCount() {
- OptionalInt maxStoreFileRefCount = this.storeEngine.getStoreFileManager()
- .getStorefiles()
+ public int getMaxCompactedStoreFileRefCount() {
+ OptionalInt maxCompactedStoreFileRefCount =
this.storeEngine.getStoreFileManager()
+ .getCompactedfiles()
.stream()
.filter(sf -> sf.getReader() != null)
.filter(HStoreFile::isHFile)
.mapToInt(HStoreFile::getRefCount)
.max();
- return maxStoreFileRefCount.isPresent() ? maxStoreFileRefCount.getAsInt()
: 0;
+ return maxCompactedStoreFileRefCount.isPresent()
+ ? maxCompactedStoreFileRefCount.getAsInt() : 0;
}
void reportArchivedFilesForQuota(List<? extends StoreFile> archivedFiles,
List<Long> fileSizes) {
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
index d2b06d0..41b515a 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
@@ -49,7 +49,7 @@ public class MetricsRegionWrapperImpl implements
MetricsRegionWrapper, Closeable
private Runnable runnable;
private long numStoreFiles;
private long storeRefCount;
- private long maxStoreFileRefCount;
+ private long maxCompactedStoreFileRefCount;
private long memstoreSize;
private long storeFileSize;
private long maxStoreFileAge;
@@ -127,8 +127,8 @@ public class MetricsRegionWrapperImpl implements
MetricsRegionWrapper, Closeable
}
@Override
- public long getMaxStoreFileRefCount() {
- return maxStoreFileRefCount;
+ public long getMaxCompactedStoreFileRefCount() {
+ return maxCompactedStoreFileRefCount;
}
@Override
@@ -234,7 +234,7 @@ public class MetricsRegionWrapperImpl implements
MetricsRegionWrapper, Closeable
public void run() {
long tempNumStoreFiles = 0;
int tempStoreRefCount = 0;
- int tempMaxStoreFileRefCount = 0;
+ int tempMaxCompactedStoreFileRefCount = 0;
long tempMemstoreSize = 0;
long tempStoreFileSize = 0;
long tempMaxStoreFileAge = 0;
@@ -249,9 +249,9 @@ public class MetricsRegionWrapperImpl implements
MetricsRegionWrapper, Closeable
tempNumStoreFiles += store.getStorefilesCount();
int currentStoreRefCount = store.getStoreRefCount();
tempStoreRefCount += currentStoreRefCount;
- int currentMaxStoreFileRefCount = store.getMaxStoreFileRefCount();
- tempMaxStoreFileRefCount = Math.max(tempMaxStoreFileRefCount,
- currentMaxStoreFileRefCount);
+ int currentMaxCompactedStoreFileRefCount =
store.getMaxCompactedStoreFileRefCount();
+ tempMaxCompactedStoreFileRefCount =
Math.max(tempMaxCompactedStoreFileRefCount,
+ currentMaxCompactedStoreFileRefCount);
tempMemstoreSize += store.getMemStoreSize().getDataSize();
tempStoreFileSize += store.getStorefilesSize();
OptionalLong storeMaxStoreFileAge = store.getMaxStoreFileAge();
@@ -279,7 +279,7 @@ public class MetricsRegionWrapperImpl implements
MetricsRegionWrapper, Closeable
numStoreFiles = tempNumStoreFiles;
storeRefCount = tempStoreRefCount;
- maxStoreFileRefCount = tempMaxStoreFileRefCount;
+ maxCompactedStoreFileRefCount = tempMaxCompactedStoreFileRefCount;
memstoreSize = tempMemstoreSize;
storeFileSize = tempStoreFileSize;
maxStoreFileAge = tempMaxStoreFileAge;
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java
index 58cb549..f170d79 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java
@@ -214,7 +214,7 @@ public class TestRegionsRecoveryChore {
Mockito.verify(hMaster, Mockito.times(0)).reopenRegions(Mockito.any(),
Mockito.anyList(),
Mockito.anyLong(), Mockito.anyLong());
- // default maxStoreFileRefCount is -1 (no regions to be reopened using AM)
+ // default maxCompactedStoreFileRefCount is -1 (no regions to be reopened
using AM)
Mockito.verify(hMaster, Mockito.times(0)).getAssignmentManager();
Mockito.verify(assignmentManager, Mockito.times(0))
.getRegionInfo(Mockito.any());
@@ -380,7 +380,7 @@ public class TestRegionsRecoveryChore {
return serverMetrics;
}
- private static RegionMetrics getRegionMetrics(byte[] regionName, int
storeRefCount) {
+ private static RegionMetrics getRegionMetrics(byte[] regionName, int
compactedStoreRefCount) {
RegionMetrics regionMetrics = new RegionMetrics() {
@Override
@@ -480,12 +480,12 @@ public class TestRegionsRecoveryChore {
@Override
public int getStoreRefCount() {
- return storeRefCount;
+ return compactedStoreRefCount;
}
@Override
- public int getMaxStoreFileRefCount() {
- return storeRefCount;
+ public int getMaxCompactedStoreFileRefCount() {
+ return compactedStoreRefCount;
}
};
@@ -610,4 +610,4 @@ public class TestRegionsRecoveryChore {
}
-}
\ No newline at end of file
+}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
index e330f4a..b6d714d 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
@@ -66,7 +66,7 @@ public class MetricsRegionWrapperStub implements
MetricsRegionWrapper {
}
@Override
- public long getMaxStoreFileRefCount() {
+ public long getMaxCompactedStoreFileRefCount() {
return 0;
}
diff --git a/src/main/asciidoc/_chapters/hbase-default.adoc
b/src/main/asciidoc/_chapters/hbase-default.adoc
index b18cd43..ebed2f2 100644
--- a/src/main/asciidoc/_chapters/hbase-default.adoc
+++ b/src/main/asciidoc/_chapters/hbase-default.adoc
@@ -2047,19 +2047,25 @@ A comma-separated list of
+
.Description
- Very large ref count on a file indicates
- that it is a ref leak on that object. Such files
- can not be removed even after it is invalidated
- via compaction. Only way to recover in such
- scenario is to reopen the region which can
- release all resources, like the refcount, leases, etc.
- This config represents Store files Ref Count threshold
- value considered for reopening regions.
- Any region with store files ref count > this value
- would be eligible for reopening by master.
- Default value -1 indicates this feature is turned off.
- Only positive integer value should be provided to enable
- this feature.
+ Very large number of ref count on a compacted
+ store file indicates that it is a ref leak
+ on that object(compacted store file).
+ Such files can not be removed after
+ it is invalidated via compaction.
+ Only way to recover in such scenario is to
+ reopen the region which can release
+ all resources, like the refcount,
+ leases, etc. This config represents Store files Ref
+ Count threshold value considered for reopening
+ regions. Any region with compacted store files
+ ref count > this value would be eligible for
+ reopening by master. Here, we get the max
+ refCount among all refCounts on all
+ compacted away store files that belong to a
+ particular region. Default value -1 indicates
+ this feature is turned off. Only positive
+ integer value should be provided to
+ enable this feature.
+
.Default
