[hbase] 01/02: HBASE-27859 HMaster.getCompactionState can happen NPE when region state is closed (#5232)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit d525629eb57551951e56f9ea35c5c50eb575de95
Author: guluo
AuthorDate: Thu Jul 6 10:32:38 2023 +0800
HBASE-27859 HMaster.getCompactionState can happen NPE when region state is
closed (#5232)
Signed-off-by: Duo Zhang
Signed-off-by: Wellington Chevreuil
(cherry picked from commit b2e2abe64bd9f3d511b8193510fe66c76ff7854c)
---
.../src/main/java/org/apache/hadoop/hbase/master/HMaster.java| 5 +
1 file changed, 5 insertions(+)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 3d59db24501..8cb40cb5880 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -4208,6 +4208,11 @@ public class HMaster extends
HBaseServerBase implements Maste
continue;
}
RegionMetrics regionMetrics =
sl.getRegionMetrics().get(regionInfo.getRegionName());
+if (regionMetrics == null) {
+ LOG.warn("Can not get compaction details for the region: {} , it may
be not online.",
+regionInfo.getRegionNameAsString());
+ continue;
+}
if (regionMetrics.getCompactionState() == CompactionState.MAJOR) {
if (compactionState == CompactionState.MINOR) {
compactionState = CompactionState.MAJOR_AND_MINOR;
[hbase] 02/02: HBASE-27845 Distinguish the mutate types of rpc error in MetricsConnection. (#5224)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 4c7698fbd378d3ac161813ef5aa54cfba44ca857
Author: Fantasy-Jay <13631435...@163.com>
AuthorDate: Thu Jul 6 10:32:08 2023 +0800
HBASE-27845 Distinguish the mutate types of rpc error in MetricsConnection.
(#5224)
Co-authored-by: fantasy <875282...@qq.com>
Co-authored-by: jay.zhu
Signed-off-by: Duo Zhang
(cherry picked from commit 047f077f0b6117985eab9d02bb10bb8c328fabfb)
---
.../hadoop/hbase/client/MetricsConnection.java | 25 --
.../hadoop/hbase/client/TestMetricsConnection.java | 39 --
2 files changed, 52 insertions(+), 12 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java
index dd3e571a8b7..8a299dc4e5c 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java
@@ -652,7 +652,28 @@ public final class MetricsConnection implements
StatisticTrackable {
concurrentCallsPerServerHist.update(callsPerServer);
}
// Update the counter that tracks RPCs by type.
-final String methodName = method.getService().getName() + "_" +
method.getName();
+StringBuilder methodName = new StringBuilder();
+
methodName.append(method.getService().getName()).append("_").append(method.getName());
+// Distinguish mutate types.
+if ("Mutate".equals(method.getName())) {
+ final MutationType type = ((MutateRequest)
param).getMutation().getMutateType();
+ switch (type) {
+case APPEND:
+ methodName.append("(Append)");
+ break;
+case DELETE:
+ methodName.append("(Delete)");
+ break;
+case INCREMENT:
+ methodName.append("(Increment)");
+ break;
+case PUT:
+ methodName.append("(Put)");
+ break;
+default:
+ methodName.append("(Unknown)");
+ }
+}
getMetric(CNT_BASE + methodName, rpcCounters, counterFactory).inc();
if (e != null) {
getMetric(FAILURE_CNT_BASE + methodName, rpcCounters,
counterFactory).inc();
@@ -729,7 +750,7 @@ public final class MetricsConnection implements
StatisticTrackable {
}
}
// Fallback to dynamic registry lookup for DDL methods.
-updateRpcGeneric(methodName, stats);
+updateRpcGeneric(methodName.toString(), stats);
}
public void incrCacheDroppingExceptions(Object exception) {
diff --git
a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java
b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java
index d70d2cf6000..2afdc7ee558 100644
---
a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java
+++
b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java
@@ -99,7 +99,7 @@ public class TestMetricsConnection {
}
@Test
- public void testMetricsWithMutiConnections() throws IOException {
+ public void testMetricsWithMultiConnections() throws IOException {
Configuration conf = new Configuration();
conf.setBoolean(MetricsConnection.CLIENT_SIDE_METRICS_ENABLED_KEY, true);
conf.set(MetricsConnection.METRICS_SCOPE_KEY, "unit-test");
@@ -178,7 +178,8 @@ public class TestMetricsConnection {
MutateRequest.newBuilder()
.setMutation(ProtobufUtil.toMutation(MutationType.PUT, new
Put(foo))).setRegion(region)
.build(),
-MetricsConnection.newCallStats(), null);
+MetricsConnection.newCallStats(),
+new CallTimeoutException("test with CallTimeoutException"));
}
final String rpcCountPrefix = "rpcCount_" +
ClientService.getDescriptor().getName() + "_";
@@ -188,20 +189,38 @@ public class TestMetricsConnection {
long metricVal;
Counter counter;
-for (String method : new String[] { "Get", "Scan", "Multi", "Mutate" }) {
+for (String method : new String[] { "Get", "Scan", "Multi" }) {
metricKey = rpcCountPrefix + method;
metricVal = METRICS.getRpcCounters().get(metricKey).getCount();
- assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal >=
loop);
+ assertEquals("metric: " + metricKey + " val: " + metricVal, metricVal,
loop);
metricKey = rpcFailureCountPrefix + method;
counter = METRICS.getRpcCounters().get(metricKey);
metricVal = (counter != null) ? counter.getCount() : 0;
- if (method.equals("Get") || method.equals("Mutate")) {
+ if (method.equals("Get")) {
// no failure
-assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal ==
0);
+assertEquals("metric: " + metricKey + "
[hbase] branch branch-3 updated (2c9d7fde54c -> 4c7698fbd37)
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a change to branch branch-3 in repository https://gitbox.apache.org/repos/asf/hbase.git from 2c9d7fde54c HBASE-27920 Skipping compact for this region if the table disable compaction (#5273) new d525629eb57 HBASE-27859 HMaster.getCompactionState can happen NPE when region state is closed (#5232) new 4c7698fbd37 HBASE-27845 Distinguish the mutate types of rpc error in MetricsConnection. (#5224) The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../hadoop/hbase/client/MetricsConnection.java | 25 -- .../hadoop/hbase/client/TestMetricsConnection.java | 39 -- .../org/apache/hadoop/hbase/master/HMaster.java| 5 +++ 3 files changed, 57 insertions(+), 12 deletions(-)
[hbase] 01/02: HBASE-27859 HMaster.getCompactionState can happen NPE when region state is closed (#5232)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit de37c4cebacfb7859783200301680f1cabc50060
Author: guluo
AuthorDate: Thu Jul 6 10:32:38 2023 +0800
HBASE-27859 HMaster.getCompactionState can happen NPE when region state is
closed (#5232)
Signed-off-by: Duo Zhang
Signed-off-by: Wellington Chevreuil
(cherry picked from commit b2e2abe64bd9f3d511b8193510fe66c76ff7854c)
---
.../src/main/java/org/apache/hadoop/hbase/master/HMaster.java| 5 +
1 file changed, 5 insertions(+)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index b61737020da..e5625820874 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -4146,6 +4146,11 @@ public class HMaster extends HRegionServer implements
MasterServices {
continue;
}
RegionMetrics regionMetrics =
sl.getRegionMetrics().get(regionInfo.getRegionName());
+if (regionMetrics == null) {
+ LOG.warn("Can not get compaction details for the region: {} , it may
be not online.",
+regionInfo.getRegionNameAsString());
+ continue;
+}
if (regionMetrics.getCompactionState() == CompactionState.MAJOR) {
if (compactionState == CompactionState.MINOR) {
compactionState = CompactionState.MAJOR_AND_MINOR;
[hbase] 02/02: HBASE-27845 Distinguish the mutate types of rpc error in MetricsConnection. (#5224)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit c82cf47354ee4970e346d4fdde45ff0273fd2097
Author: Fantasy-Jay <13631435...@163.com>
AuthorDate: Thu Jul 6 10:32:08 2023 +0800
HBASE-27845 Distinguish the mutate types of rpc error in MetricsConnection.
(#5224)
Co-authored-by: fantasy <875282...@qq.com>
Co-authored-by: jay.zhu
Signed-off-by: Duo Zhang
(cherry picked from commit 047f077f0b6117985eab9d02bb10bb8c328fabfb)
---
.../hadoop/hbase/client/MetricsConnection.java | 25 --
.../hadoop/hbase/client/TestMetricsConnection.java | 39 --
2 files changed, 52 insertions(+), 12 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java
index 1d337372927..89536d430bf 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java
@@ -700,7 +700,28 @@ public final class MetricsConnection implements
StatisticTrackable {
concurrentCallsPerServerHist.update(callsPerServer);
}
// Update the counter that tracks RPCs by type.
-final String methodName = method.getService().getName() + "_" +
method.getName();
+StringBuilder methodName = new StringBuilder();
+
methodName.append(method.getService().getName()).append("_").append(method.getName());
+// Distinguish mutate types.
+if ("Mutate".equals(method.getName())) {
+ final MutationType type = ((MutateRequest)
param).getMutation().getMutateType();
+ switch (type) {
+case APPEND:
+ methodName.append("(Append)");
+ break;
+case DELETE:
+ methodName.append("(Delete)");
+ break;
+case INCREMENT:
+ methodName.append("(Increment)");
+ break;
+case PUT:
+ methodName.append("(Put)");
+ break;
+default:
+ methodName.append("(Unknown)");
+ }
+}
getMetric(CNT_BASE + methodName, rpcCounters, counterFactory).inc();
if (e != null) {
getMetric(FAILURE_CNT_BASE + methodName, rpcCounters,
counterFactory).inc();
@@ -777,7 +798,7 @@ public final class MetricsConnection implements
StatisticTrackable {
}
}
// Fallback to dynamic registry lookup for DDL methods.
-updateRpcGeneric(methodName, stats);
+updateRpcGeneric(methodName.toString(), stats);
}
public void incrCacheDroppingExceptions(Object exception) {
diff --git
a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java
b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java
index 66d81599427..469ecd4e10e 100644
---
a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java
+++
b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java
@@ -102,7 +102,7 @@ public class TestMetricsConnection {
}
@Test
- public void testMetricsWithMutiConnections() throws IOException {
+ public void testMetricsWithMultiConnections() throws IOException {
Configuration conf = new Configuration();
conf.setBoolean(MetricsConnection.CLIENT_SIDE_METRICS_ENABLED_KEY, true);
conf.set(MetricsConnection.METRICS_SCOPE_KEY, "unit-test");
@@ -205,7 +205,8 @@ public class TestMetricsConnection {
MutateRequest.newBuilder()
.setMutation(ProtobufUtil.toMutation(MutationType.PUT, new
Put(foo))).setRegion(region)
.build(),
-MetricsConnection.newCallStats(), null);
+MetricsConnection.newCallStats(),
+new CallTimeoutException("test with CallTimeoutException"));
}
final String rpcCountPrefix = "rpcCount_" +
ClientService.getDescriptor().getName() + "_";
@@ -215,20 +216,38 @@ public class TestMetricsConnection {
long metricVal;
Counter counter;
-for (String method : new String[] { "Get", "Scan", "Multi", "Mutate" }) {
+for (String method : new String[] { "Get", "Scan", "Multi" }) {
metricKey = rpcCountPrefix + method;
metricVal = METRICS.getRpcCounters().get(metricKey).getCount();
- assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal >=
loop);
+ assertEquals("metric: " + metricKey + " val: " + metricVal, metricVal,
loop);
metricKey = rpcFailureCountPrefix + method;
counter = METRICS.getRpcCounters().get(metricKey);
metricVal = (counter != null) ? counter.getCount() : 0;
- if (method.equals("Get") || method.equals("Mutate")) {
+ if (method.equals("Get")) {
// no failure
-assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal ==
0);
+assertEquals("metric: " + metricKey +
[hbase] branch branch-2 updated (c4a1b447d7b -> c82cf47354e)
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a change to branch branch-2 in repository https://gitbox.apache.org/repos/asf/hbase.git from c4a1b447d7b HBASE-27920 Skipping compact for this region if the table disable compaction (#5273) new de37c4cebac HBASE-27859 HMaster.getCompactionState can happen NPE when region state is closed (#5232) new c82cf47354e HBASE-27845 Distinguish the mutate types of rpc error in MetricsConnection. (#5224) The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../hadoop/hbase/client/MetricsConnection.java | 25 -- .../hadoop/hbase/client/TestMetricsConnection.java | 39 -- .../org/apache/hadoop/hbase/master/HMaster.java| 5 +++ 3 files changed, 57 insertions(+), 12 deletions(-)
[hbase] branch branch-2.5 updated: HBASE-27859 HMaster.getCompactionState can happen NPE when region state is closed (#5232)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.5 by this push:
new 0ffa7518e08 HBASE-27859 HMaster.getCompactionState can happen NPE when
region state is closed (#5232)
0ffa7518e08 is described below
commit 0ffa7518e08781c4700d81080d722733b486b51a
Author: guluo
AuthorDate: Thu Jul 6 10:32:38 2023 +0800
HBASE-27859 HMaster.getCompactionState can happen NPE when region state is
closed (#5232)
Signed-off-by: Duo Zhang
Signed-off-by: Wellington Chevreuil
(cherry picked from commit b2e2abe64bd9f3d511b8193510fe66c76ff7854c)
---
.../src/main/java/org/apache/hadoop/hbase/master/HMaster.java| 5 +
1 file changed, 5 insertions(+)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index a9b70847071..f8a64908783 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -4127,6 +4127,11 @@ public class HMaster extends HRegionServer implements
MasterServices {
continue;
}
RegionMetrics regionMetrics =
sl.getRegionMetrics().get(regionInfo.getRegionName());
+if (regionMetrics == null) {
+ LOG.warn("Can not get compaction details for the region: {} , it may
be not online.",
+regionInfo.getRegionNameAsString());
+ continue;
+}
if (regionMetrics.getCompactionState() == CompactionState.MAJOR) {
if (compactionState == CompactionState.MINOR) {
compactionState = CompactionState.MAJOR_AND_MINOR;
[hbase] branch branch-2.4 updated: HBASE-27859 HMaster.getCompactionState can happen NPE when region state is closed (#5232)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new d8b9d82e27b HBASE-27859 HMaster.getCompactionState can happen NPE when
region state is closed (#5232)
d8b9d82e27b is described below
commit d8b9d82e27b52a845308e7cfb659b9056be2fde3
Author: guluo
AuthorDate: Thu Jul 6 10:32:38 2023 +0800
HBASE-27859 HMaster.getCompactionState can happen NPE when region state is
closed (#5232)
Signed-off-by: Duo Zhang
Signed-off-by: Wellington Chevreuil
(cherry picked from commit b2e2abe64bd9f3d511b8193510fe66c76ff7854c)
---
.../src/main/java/org/apache/hadoop/hbase/master/HMaster.java| 5 +
1 file changed, 5 insertions(+)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 9209ebaafd4..3e03cbfda09 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -3799,6 +3799,11 @@ public class HMaster extends HRegionServer implements
MasterServices {
continue;
}
RegionMetrics regionMetrics =
sl.getRegionMetrics().get(regionInfo.getRegionName());
+if (regionMetrics == null) {
+ LOG.warn("Can not get compaction details for the region: {} , it may
be not online.",
+regionInfo.getRegionNameAsString());
+ continue;
+}
if (regionMetrics.getCompactionState() == CompactionState.MAJOR) {
if (compactionState == CompactionState.MINOR) {
compactionState = CompactionState.MAJOR_AND_MINOR;
[hbase] branch master updated (047f077f0b6 -> b2e2abe64bd)
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git from 047f077f0b6 HBASE-27845 Distinguish the mutate types of rpc error in MetricsConnection. (#5224) add b2e2abe64bd HBASE-27859 HMaster.getCompactionState can happen NPE when region state is closed (#5232) No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/hadoop/hbase/master/HMaster.java| 5 + 1 file changed, 5 insertions(+)
[hbase] branch master updated (d1f29d06ece -> 047f077f0b6)
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git from d1f29d06ece HBASE-27920 Skipping compact for this region if the table disable compaction (#5273) add 047f077f0b6 HBASE-27845 Distinguish the mutate types of rpc error in MetricsConnection. (#5224) No new revisions were added by this update. Summary of changes: .../hadoop/hbase/client/MetricsConnection.java | 25 -- .../hadoop/hbase/client/TestMetricsConnection.java | 39 -- 2 files changed, 52 insertions(+), 12 deletions(-)
[hbase] branch branch-3 updated: HBASE-27920 Skipping compact for this region if the table disable compaction (#5273)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new 2c9d7fde54c HBASE-27920 Skipping compact for this region if the table
disable compaction (#5273)
2c9d7fde54c is described below
commit 2c9d7fde54ce83bf3c3365ca448b2f3298c32129
Author: guluo
AuthorDate: Thu Jul 6 10:19:32 2023 +0800
HBASE-27920 Skipping compact for this region if the table disable
compaction (#5273)
Signed-off-by: Duo Zhang
Signed-off-by: Wellington Chevreuil
(cherry picked from commit d1f29d06ece649c32caa2ae5551e9a67a540014a)
---
.../java/org/apache/hadoop/hbase/regionserver/HRegionServer.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 1c5940ca9ae..f9f84118106 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1638,8 +1638,9 @@ public class HRegionServer extends
HBaseServerBase
@Override
protected void chore() {
for (Region r : this.instance.onlineRegions.values()) {
-// Skip compaction if region is read only
-if (r == null || r.isReadOnly()) {
+// If region is read only or compaction is disabled at table level,
there's no need to
+// iterate through region's stores
+if (r == null || r.isReadOnly() ||
!r.getTableDescriptor().isCompactionEnabled()) {
continue;
}
[hbase] branch branch-2 updated (7640557f3b3 -> c4a1b447d7b)
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a change to branch branch-2 in repository https://gitbox.apache.org/repos/asf/hbase.git from 7640557f3b3 HBASE-27942 Update the description about hbase.hstore.compactionThreshold (#5302) add c4a1b447d7b HBASE-27920 Skipping compact for this region if the table disable compaction (#5273) No new revisions were added by this update. Summary of changes: .../main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
[hbase] branch branch-2.5 updated: HBASE-27920 Skipping compact for this region if the table disable compaction (#5273)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.5 by this push:
new 0d8c1609b7b HBASE-27920 Skipping compact for this region if the table
disable compaction (#5273)
0d8c1609b7b is described below
commit 0d8c1609b7bd29a198f486099eedd08cff45b6fc
Author: guluo
AuthorDate: Thu Jul 6 10:19:32 2023 +0800
HBASE-27920 Skipping compact for this region if the table disable
compaction (#5273)
Signed-off-by: Duo Zhang
Signed-off-by: Wellington Chevreuil
(cherry picked from commit d1f29d06ece649c32caa2ae5551e9a67a540014a)
---
.../main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 7bf543927ad..c33997c19b6 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1920,7 +1920,9 @@ public class HRegionServer extends Thread
@Override
protected void chore() {
for (Region r : this.instance.onlineRegions.values()) {
-if (r == null) {
+// If region is read only or compaction is disabled at table level,
there's no need to
+// iterate through region's stores
+if (r == null || r.isReadOnly() ||
!r.getTableDescriptor().isCompactionEnabled()) {
continue;
}
HRegion hr = (HRegion) r;
[hbase] branch branch-2.4 updated: HBASE-27920 Skipping compact for this region if the table disable compaction (#5273)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 94f84244cb2 HBASE-27920 Skipping compact for this region if the table
disable compaction (#5273)
94f84244cb2 is described below
commit 94f84244cb28b3f15ea890acbd52104965a3d925
Author: guluo
AuthorDate: Thu Jul 6 10:19:32 2023 +0800
HBASE-27920 Skipping compact for this region if the table disable
compaction (#5273)
Signed-off-by: Duo Zhang
Signed-off-by: Wellington Chevreuil
(cherry picked from commit d1f29d06ece649c32caa2ae5551e9a67a540014a)
---
.../main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 2a4816777f5..62e781de0d1 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1846,7 +1846,9 @@ public class HRegionServer extends Thread
@Override
protected void chore() {
for (Region r : this.instance.onlineRegions.values()) {
-if (r == null) {
+// If region is read only or compaction is disabled at table level,
there's no need to
+// iterate through region's stores
+if (r == null || r.isReadOnly() ||
!r.getTableDescriptor().isCompactionEnabled()) {
continue;
}
HRegion hr = (HRegion) r;
[hbase] branch master updated: HBASE-27920 Skipping compact for this region if the table disable compaction (#5273)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new d1f29d06ece HBASE-27920 Skipping compact for this region if the table
disable compaction (#5273)
d1f29d06ece is described below
commit d1f29d06ece649c32caa2ae5551e9a67a540014a
Author: guluo
AuthorDate: Thu Jul 6 10:19:32 2023 +0800
HBASE-27920 Skipping compact for this region if the table disable
compaction (#5273)
Signed-off-by: Duo Zhang
Signed-off-by: Wellington Chevreuil
---
.../java/org/apache/hadoop/hbase/regionserver/HRegionServer.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 1c5940ca9ae..f9f84118106 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1638,8 +1638,9 @@ public class HRegionServer extends
HBaseServerBase
@Override
protected void chore() {
for (Region r : this.instance.onlineRegions.values()) {
-// Skip compaction if region is read only
-if (r == null || r.isReadOnly()) {
+// If region is read only or compaction is disabled at table level,
there's no need to
+// iterate through region's stores
+if (r == null || r.isReadOnly() ||
!r.getTableDescriptor().isCompactionEnabled()) {
continue;
}
[hbase-site] branch asf-site updated: INFRA-10751 Empty commit
This is an automated email from the ASF dual-hosted git repository. git-site-role pushed a commit to branch asf-site in repository https://gitbox.apache.org/repos/asf/hbase-site.git The following commit(s) were added to refs/heads/asf-site by this push: new 9c1f951648a INFRA-10751 Empty commit 9c1f951648a is described below commit 9c1f951648a8fd6a879d03771c15e1ae907e865d Author: jenkins AuthorDate: Wed Jul 5 14:42:59 2023 + INFRA-10751 Empty commit
[hbase] branch branch-3 updated: HBASE-27942 Update the description about hbase.hstore.compactionThreshold (#5302)
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch branch-3 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-3 by this push: new f8773784818 HBASE-27942 Update the description about hbase.hstore.compactionThreshold (#5302) f8773784818 is described below commit f87737848182e04bb26b6cc3e7232f35d8e4be51 Author: guluo AuthorDate: Wed Jul 5 22:07:49 2023 +0800 HBASE-27942 Update the description about hbase.hstore.compactionThreshold (#5302) Signed-off-by: Duo Zhang (cherry picked from commit d8447d9fef57076c479f7ced6564a0cfb582cf3e) --- hbase-common/src/main/resources/hbase-default.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index a4ee0c8b20b..d2ecef6bda3 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -812,7 +812,7 @@ possible configurations would overwhelm and obscure the important. hbase.hstore.compactionThreshold 3 - If more than this number of StoreFiles exist in any one Store + If more than or equal to this number of StoreFiles exist in any one Store (one StoreFile is written per flush of MemStore), a compaction is run to rewrite all StoreFiles into a single StoreFile. Larger values delay compaction, but when compaction does occur, it takes longer to complete.
[hbase] branch branch-2 updated: HBASE-27942 Update the description about hbase.hstore.compactionThreshold (#5302)
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch branch-2 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2 by this push: new 7640557f3b3 HBASE-27942 Update the description about hbase.hstore.compactionThreshold (#5302) 7640557f3b3 is described below commit 7640557f3b3ecf383514d0974957a1eee104103f Author: guluo AuthorDate: Wed Jul 5 22:07:49 2023 +0800 HBASE-27942 Update the description about hbase.hstore.compactionThreshold (#5302) Signed-off-by: Duo Zhang (cherry picked from commit d8447d9fef57076c479f7ced6564a0cfb582cf3e) --- hbase-common/src/main/resources/hbase-default.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index f48eb1a6203..39b98ec65d9 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -802,7 +802,7 @@ possible configurations would overwhelm and obscure the important. hbase.hstore.compactionThreshold 3 - If more than this number of StoreFiles exist in any one Store + If more than or equal to this number of StoreFiles exist in any one Store (one StoreFile is written per flush of MemStore), a compaction is run to rewrite all StoreFiles into a single StoreFile. Larger values delay compaction, but when compaction does occur, it takes longer to complete.
[hbase] branch branch-2.5 updated: HBASE-27942 Update the description about hbase.hstore.compactionThreshold (#5302)
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch branch-2.5 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2.5 by this push: new c926353096c HBASE-27942 Update the description about hbase.hstore.compactionThreshold (#5302) c926353096c is described below commit c926353096cc8e3ac0af81c03fbf94c2a562106f Author: guluo AuthorDate: Wed Jul 5 22:07:49 2023 +0800 HBASE-27942 Update the description about hbase.hstore.compactionThreshold (#5302) Signed-off-by: Duo Zhang (cherry picked from commit d8447d9fef57076c479f7ced6564a0cfb582cf3e) --- hbase-common/src/main/resources/hbase-default.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index 270d097e024..ecbda46551f 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -802,7 +802,7 @@ possible configurations would overwhelm and obscure the important. hbase.hstore.compactionThreshold 3 - If more than this number of StoreFiles exist in any one Store + If more than or equal to this number of StoreFiles exist in any one Store (one StoreFile is written per flush of MemStore), a compaction is run to rewrite all StoreFiles into a single StoreFile. Larger values delay compaction, but when compaction does occur, it takes longer to complete.
[hbase] branch branch-2.4 updated: HBASE-27942 Update the description about hbase.hstore.compactionThreshold (#5302)
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch branch-2.4 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2.4 by this push: new 46368c9ecf4 HBASE-27942 Update the description about hbase.hstore.compactionThreshold (#5302) 46368c9ecf4 is described below commit 46368c9ecf4cab8e8c76cbb67312e4eaeba15dca Author: guluo AuthorDate: Wed Jul 5 22:07:49 2023 +0800 HBASE-27942 Update the description about hbase.hstore.compactionThreshold (#5302) Signed-off-by: Duo Zhang (cherry picked from commit d8447d9fef57076c479f7ced6564a0cfb582cf3e) --- hbase-common/src/main/resources/hbase-default.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index 269c99dba27..c84eb8d183d 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -784,7 +784,7 @@ possible configurations would overwhelm and obscure the important. hbase.hstore.compactionThreshold 3 - If more than this number of StoreFiles exist in any one Store + If more than or equal to this number of StoreFiles exist in any one Store (one StoreFile is written per flush of MemStore), a compaction is run to rewrite all StoreFiles into a single StoreFile. Larger values delay compaction, but when compaction does occur, it takes longer to complete.
[hbase] branch master updated: HBASE-27663 ChaosMonkey documentation enhancements (#5172)
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/master by this push: new 2ca012f6f86 HBASE-27663 ChaosMonkey documentation enhancements (#5172) 2ca012f6f86 is described below commit 2ca012f6f868bcda7d0faa7d343a4627c919fb86 Author: Yash Dodeja AuthorDate: Wed Jul 5 19:43:25 2023 +0530 HBASE-27663 ChaosMonkey documentation enhancements (#5172) Signed-off-by: Duo Zhang --- src/main/asciidoc/_chapters/developer.adoc | 60 +- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/src/main/asciidoc/_chapters/developer.adoc b/src/main/asciidoc/_chapters/developer.adoc index 20f96921b1d..ea327fb3e25 100644 --- a/src/main/asciidoc/_chapters/developer.adoc +++ b/src/main/asciidoc/_chapters/developer.adoc @@ -1759,7 +1759,7 @@ following example runs ChaosMonkey with the default configuration: [source,bash] -$ bin/hbase org.apache.hadoop.hbase.util.ChaosMonkey +$ bin/hbase org.apache.hadoop.hbase.chaos.util.ChaosMonkeyRunner 12/11/19 23:21:57 INFO util.ChaosMonkey: Using ChaosMonkey Policy: class org.apache.hadoop.hbase.util.ChaosMonkey$PeriodicRandomActionPolicy, period:6 12/11/19 23:21:57 INFO util.ChaosMonkey: Sleeping for 26953 to add jitter @@ -1801,6 +1801,64 @@ $ bin/hbase org.apache.hadoop.hbase.util.ChaosMonkey The output indicates that ChaosMonkey started the default `PeriodicRandomActionPolicy` policy, which is configured with all the available actions. It chose to run `RestartActiveMaster` and `RestartRandomRs` actions. + ChaosMonkey without SSH + +Chaos monkey can be run without SSH using the Chaos service and ZNode cluster manager. HBase ships +with many cluster managers, available in the `hbase-it/src/test/java/org/apache/hadoop/hbase/` directory. + +Set the following property in hbase configuration to switch to `ZNodeClusterManager`: +`hbase.it.clustermanager.class=org.apache.hadoop.hbase.ZNodeClusterManager` + +Start chaos agent on all hosts where you want to test chaos scenarios. + +[source,bash] + +$ bin/hbase org.apache.hadoop.hbase.chaos.ChaosService -c start + +Start chaos monkey runner from any one host, preferrably an edgenode. +An example log while running chaos monkey with default policy PeriodicRandomActionPolicy is shown below. +Command Options: + -c Name of extra configurations file to find on CLASSPATH + -m,--monkey Which chaos monkey to run + -monkeyPropsThe properties file for specifying chaos monkey properties. + -tableName Table name in the test to run chaos monkey against + -familyName Family name in the test to run chaos monkey against + +[source,bash] + +$ bin/hbase org.apache.hadoop.hbase.chaos.util.ChaosMonkeyRunner + +INFO [main] hbase.HBaseCommonTestingUtility: Instantiating org.apache.hadoop.hbase.ZNodeClusterManager +INFO [ReadOnlyZKClient-host1.example.com:2181,host2.example.com:2181,host3.example.com:2181@0x003d43fe] zookeeper.ZooKeeper: Initiating client connection, connectString=host1.example.com:2181,host2.example.com:2181,host3.example.com:2181 sessionTimeout=9 watcher=org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$19/2106254492@1a39cf8 +INFO [ReadOnlyZKClient-host1.example.com:2181,host2.example.com:2181,host3.example.com:2181@0x003d43fe] zookeeper.ClientCnxnSocket: jute.maxbuffer value is 4194304 Bytes +INFO [ReadOnlyZKClient-host1.example.com:2181,host2.example.com:2181,host3.example.com:2181@0x003d43fe] zookeeper.ClientCnxn: zookeeper.request.timeout value is 0. feature enabled= +INFO [ReadOnlyZKClient-host1.example.com:2181,host2.example.com:2181,host3.example.com:2181@0x003d43fe-SendThread(host2.example.com:2181)] zookeeper.ClientCnxn: Opening socket connection to server host2.example.com/10.20.30.40:2181. Will not attempt to authenticate using SASL (unknown error) +INFO [ReadOnlyZKClient-host1.example.com:2181,host2.example.com:2181,host3.example.com:2181@0x003d43fe-SendThread(host2.example.com:2181)] zookeeper.ClientCnxn: Socket connection established, initiating session, client: /10.20.30.40:35164, server: host2.example.com/10.20.30.40:2181 +INFO [ReadOnlyZKClient-host1.example.com:2181,host2.example.com:2181,host3.example.com:2181@0x003d43fe-SendThread(host2.example.com:2181)] zookeeper.ClientCnxn: Session establishment complete on server host2.example.com/10.20.30.40:2181, sessionid = 0x101de9204670877, negotiated timeout = 6 +INFO [main] policies.Policy: Using ChaosMonkey Policy class org.apache.hadoop.hbase.chaos.policies.PeriodicRandomActionPolicy, period=6 ms + [ChaosMonkey-2] policies.Policy: Sleeping for 93741 ms to add jitter +INFO [ChaosMonkey-0] policies.Policy: Sleeping for 9752 ms to add jitter +INFO [ChaosMonkey-1] policies.Policy: Sleeping for 65562 ms to add jitter +INFO [ChaosMonkey-3]
[hbase] branch master updated: HBASE-27942 Update the description about hbase.hstore.compactionThreshold (#5302)
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/master by this push: new d8447d9fef5 HBASE-27942 Update the description about hbase.hstore.compactionThreshold (#5302) d8447d9fef5 is described below commit d8447d9fef57076c479f7ced6564a0cfb582cf3e Author: guluo AuthorDate: Wed Jul 5 22:07:49 2023 +0800 HBASE-27942 Update the description about hbase.hstore.compactionThreshold (#5302) Signed-off-by: Duo Zhang --- hbase-common/src/main/resources/hbase-default.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index a4ee0c8b20b..d2ecef6bda3 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -812,7 +812,7 @@ possible configurations would overwhelm and obscure the important. hbase.hstore.compactionThreshold 3 - If more than this number of StoreFiles exist in any one Store + If more than or equal to this number of StoreFiles exist in any one Store (one StoreFile is written per flush of MemStore), a compaction is run to rewrite all StoreFiles into a single StoreFile. Larger values delay compaction, but when compaction does occur, it takes longer to complete.
