[hbase] branch master updated: HBASE-24511 Ability to configure timeout between RPC retry to RS from master (#1861)
This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 1eb159b HBASE-24511 Ability to configure timeout between RPC retry to
RS from master (#1861)
1eb159b is described below
commit 1eb159b455075f92e4e8ed3de96a599d035ccb1a
Author: sguggilam
AuthorDate: Wed Jun 10 00:23:08 2020 -0700
HBASE-24511 Ability to configure timeout between RPC retry to RS from
master (#1861)
Signed-off-by: Viraj Jasani
---
.../master/procedure/RSProcedureDispatcher.java| 17 +++---
.../TestRegionServerReportForDuty.java | 38 +-
2 files changed, 50 insertions(+), 5 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
index 72d504e..1942ed6 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
@@ -234,12 +234,19 @@ public class RSProcedureDispatcher
private int numberOfAttemptsSoFar = 0;
private long maxWaitTime = -1;
+private final long rsRpcRetryInterval;
+private static final String RS_RPC_RETRY_INTERVAL_CONF_KEY =
+"hbase.regionserver.rpc.retry.interval";
+private static final int DEFAULT_RS_RPC_RETRY_INTERVAL = 100;
+
private ExecuteProceduresRequest.Builder request = null;
public ExecuteProceduresRemoteCall(final ServerName serverName,
final Set remoteProcedures) {
this.serverName = serverName;
this.remoteProcedures = remoteProcedures;
+ this.rsRpcRetryInterval =
master.getConfiguration().getLong(RS_RPC_RETRY_INTERVAL_CONF_KEY,
+DEFAULT_RS_RPC_RETRY_INTERVAL);
}
private AsyncRegionServerAdmin getRsAdmin() throws IOException {
@@ -259,8 +266,8 @@ public class RSProcedureDispatcher
LOG.warn("Waiting a little before retrying {}, try={}, can wait up
to {}ms",
serverName, numberOfAttemptsSoFar, remainingTime);
numberOfAttemptsSoFar++;
- // Retry every 100ms up to maximum wait time.
- submitTask(this, 100, TimeUnit.MILLISECONDS);
+ // Retry every rsRpcRetryInterval millis up to maximum wait time.
+ submitTask(this, rsRpcRetryInterval, TimeUnit.MILLISECONDS);
return true;
}
LOG.warn("{} is throwing ServerNotRunningYetException for {}ms; trying
another server",
@@ -305,10 +312,12 @@ public class RSProcedureDispatcher
numberOfAttemptsSoFar++;
// Add some backoff here as the attempts rise otherwise if a stuck
condition, will fill logs
// with failed attempts. None of our backoff classes -- RetryCounter or
ClientBackoffPolicy
- // -- fit here nicely so just do something simple; increment by 100ms *
retry^2 on each try
+ // -- fit here nicely so just do something simple; increment by
rsRpcRetryInterval millis *
+ // retry^2 on each try
// up to max of 10 seconds (don't want to back off too much in case of
situation change).
submitTask(this,
-Math.min(100 * (this.numberOfAttemptsSoFar *
this.numberOfAttemptsSoFar), 10 * 1000),
+Math.min(rsRpcRetryInterval * (this.numberOfAttemptsSoFar *
this.numberOfAttemptsSoFar),
+ 10 * 1000),
TimeUnit.MILLISECONDS);
return true;
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java
index dd1c085..0e1c280 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java
@@ -22,7 +22,8 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.io.StringWriter;
-
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
@@ -38,6 +39,7 @@ import org.apache.hadoop.hbase.master.ServerManager;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
+import org.apache.hadoop.hbase.util.Threads;
import org.apache.log4j.Appender;
import org.apache.log4j.Layout;
import org.apache.log4j.PatternLayout;
@@ -222,6 +224,40 @@ public class TestRegionServerReportForDuty {
[hbase] branch branch-2 updated: HBASE-24524 SyncTable logging improvements (#1878)
This is an automated email from the ASF dual-hosted git repository.
wchevreuil pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new ef984cb HBASE-24524 SyncTable logging improvements (#1878)
ef984cb is described below
commit ef984cb18eb21f195b8b32a3fccd61cdaf4ac037
Author: Wellington Ramos Chevreuil
AuthorDate: Wed Jun 10 09:36:30 2020 +0100
HBASE-24524 SyncTable logging improvements (#1878)
Signed-off-by: Jan Hentschel
Signed-off-by: Viraj Jasani
(cherry picked from commit d8247ebae347915c3fbc73f8ab6c39b0984826b0)
---
.../java/org/apache/hadoop/hbase/mapreduce/SyncTable.java| 12 ++--
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/SyncTable.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/SyncTable.java
index cf9241d..6722501 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/SyncTable.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/SyncTable.java
@@ -363,16 +363,16 @@ public class SyncTable extends Configured implements Tool
{
boolean rowMatched;
int rowComparison = compareRowKeys(nextSourceRow, nextTargetRow);
if (rowComparison < 0) {
- if (LOG.isInfoEnabled()) {
-LOG.info("Target missing row: " + Bytes.toHex(nextSourceRow));
+ if (LOG.isDebugEnabled()) {
+LOG.debug("Target missing row: " + Bytes.toString(nextSourceRow));
}
context.getCounter(Counter.TARGETMISSINGROWS).increment(1);
rowMatched = syncRowCells(context, nextSourceRow, sourceCells,
EMPTY_CELL_SCANNER);
nextSourceRow = sourceCells.nextRow(); // advance only source to
next row
} else if (rowComparison > 0) {
- if (LOG.isInfoEnabled()) {
-LOG.info("Source missing row: " + Bytes.toHex(nextTargetRow));
+ if (LOG.isDebugEnabled()) {
+LOG.debug("Source missing row: " + Bytes.toString(nextTargetRow));
}
context.getCounter(Counter.SOURCEMISSINGROWS).increment(1);
@@ -557,10 +557,10 @@ public class SyncTable extends Configured implements Tool
{
if (LOG.isDebugEnabled()) {
LOG.debug("Different values: ");
LOG.debug(" source cell: " + sourceCell
- + " value: " + Bytes.toHex(sourceCell.getValueArray(),
+ + " value: " + Bytes.toString(sourceCell.getValueArray(),
sourceCell.getValueOffset(),
sourceCell.getValueLength()));
LOG.debug(" target cell: " + targetCell
- + " value: " + Bytes.toHex(targetCell.getValueArray(),
+ + " value: " + Bytes.toString(targetCell.getValueArray(),
targetCell.getValueOffset(),
targetCell.getValueLength()));
}
context.getCounter(Counter.DIFFERENTCELLVALUES).increment(1);
[hbase-site] branch asf-site updated: INFRA-10751 Empty commit
This is an automated email from the ASF dual-hosted git repository. git-site-role pushed a commit to branch asf-site in repository https://gitbox.apache.org/repos/asf/hbase-site.git The following commit(s) were added to refs/heads/asf-site by this push: new 7eac3c5 INFRA-10751 Empty commit 7eac3c5 is described below commit 7eac3c53659ec3a8b24a52e5e8b0f127c317f181 Author: jenkins AuthorDate: Wed Jun 10 14:47:37 2020 + INFRA-10751 Empty commit
[hbase] branch branch-1 updated: HBASE-24184 Backport HBASE-23896 to branch-1: Snapshot owner cannot delete snapshot when ACL is enabled and Kerberos is not enabled (#1510)
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new e07aaf7 HBASE-24184 Backport HBASE-23896 to branch-1: Snapshot owner
cannot delete snapshot when ACL is enabled and Kerberos is not enabled (#1510)
e07aaf7 is described below
commit e07aaf7fef7113433bdaf400c7b4b4b8280453df
Author: thangTang
AuthorDate: Wed Jun 10 16:12:36 2020 +0800
HBASE-24184 Backport HBASE-23896 to branch-1: Snapshot owner cannot delete
snapshot when ACL is enabled and Kerberos is not enabled (#1510)
Signed-off-by: binlijin
---
.../hadoop/hbase/master/snapshot/SnapshotManager.java| 3 ++-
.../apache/hadoop/hbase/client/TestSnapshotWithAcl.java | 16
2 files changed, 18 insertions(+), 1 deletion(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
index fff0ec9..7d25112 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
@@ -607,7 +607,8 @@ public class SnapshotManager extends MasterProcedureManager
implements Stoppable
builder.setVersion(SnapshotDescriptionUtils.SNAPSHOT_LAYOUT_VERSION);
}
User user = RpcServer.getRequestUser();
-if (User.isHBaseSecurityEnabled(master.getConfiguration()) && user !=
null) {
+if (master.getConfiguration().
+ getBoolean(User.HBASE_SECURITY_AUTHORIZATION_CONF_KEY, false) && user !=
null) {
builder.setOwner(user.getShortName());
}
snapshot = builder.build();
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotWithAcl.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotWithAcl.java
index b3f137b..9e33965 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotWithAcl.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotWithAcl.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hbase.client;
+import static org.junit.Assert.assertNotSame;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Coprocessor;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -26,6 +28,7 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.access.AccessControlConstants;
import org.apache.hadoop.hbase.security.access.AccessControlLists;
@@ -42,6 +45,7 @@ import org.junit.Test;
import org.junit.experimental.categories.Category;
import java.io.IOException;
+import java.util.List;
import java.util.UUID;
@Category(MediumTests.class)
@@ -243,4 +247,16 @@ public class TestSnapshotWithAcl extends SecureTestUtil {
verifyAllowed(new AccessWriteAction(TEST_TABLE), USER_OWNER, USER_RW);
verifyDenied(new AccessWriteAction(TEST_TABLE), USER_RO, USER_NONE);
}
+
+ @Test
+ public void testListSnapshot() throws Exception {
+String snapshotName1 = UUID.randomUUID().toString();
+admin.snapshot(snapshotName1, TEST_TABLE);
+List snapshotDescriptions =
admin.listSnapshots();
+for (HBaseProtos.SnapshotDescription snapshotDescription:
+ snapshotDescriptions) {
+ assertNotSame(snapshotDescription.getOwner(), "");
+}
+admin.deleteSnapshot(snapshotName1);
+ }
}
[hbase] branch branch-2.3 updated: HBASE-24511 Ability to configure timeout between RPC retry to RS from master (#1861)
This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.3 by this push:
new 5989de4 HBASE-24511 Ability to configure timeout between RPC retry to
RS from master (#1861)
5989de4 is described below
commit 5989de440220c7859fa1e89f4e27bcfb2b8a071e
Author: sguggilam
AuthorDate: Wed Jun 10 00:23:08 2020 -0700
HBASE-24511 Ability to configure timeout between RPC retry to RS from
master (#1861)
Signed-off-by: Viraj Jasani
---
.../master/procedure/RSProcedureDispatcher.java| 17 +++---
.../TestRegionServerReportForDuty.java | 38 +-
2 files changed, 50 insertions(+), 5 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
index 74e00f5..3b4ee4c 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
@@ -235,12 +235,19 @@ public class RSProcedureDispatcher
private int numberOfAttemptsSoFar = 0;
private long maxWaitTime = -1;
+private final long rsRpcRetryInterval;
+private static final String RS_RPC_RETRY_INTERVAL_CONF_KEY =
+"hbase.regionserver.rpc.retry.interval";
+private static final int DEFAULT_RS_RPC_RETRY_INTERVAL = 100;
+
private ExecuteProceduresRequest.Builder request = null;
public ExecuteProceduresRemoteCall(final ServerName serverName,
final Set remoteProcedures) {
this.serverName = serverName;
this.remoteProcedures = remoteProcedures;
+ this.rsRpcRetryInterval =
master.getConfiguration().getLong(RS_RPC_RETRY_INTERVAL_CONF_KEY,
+DEFAULT_RS_RPC_RETRY_INTERVAL);
}
private AdminService.BlockingInterface getRsAdmin() throws IOException {
@@ -265,8 +272,8 @@ public class RSProcedureDispatcher
LOG.warn("Waiting a little before retrying {}, try={}, can wait up
to {}ms",
serverName, numberOfAttemptsSoFar, remainingTime);
numberOfAttemptsSoFar++;
- // Retry every 100ms up to maximum wait time.
- submitTask(this, 100, TimeUnit.MILLISECONDS);
+ // Retry every rsRpcRetryInterval millis up to maximum wait time.
+ submitTask(this, rsRpcRetryInterval, TimeUnit.MILLISECONDS);
return true;
}
LOG.warn("{} is throwing ServerNotRunningYetException for {}ms; trying
another server",
@@ -311,10 +318,12 @@ public class RSProcedureDispatcher
numberOfAttemptsSoFar++;
// Add some backoff here as the attempts rise otherwise if a stuck
condition, will fill logs
// with failed attempts. None of our backoff classes -- RetryCounter or
ClientBackoffPolicy
- // -- fit here nicely so just do something simple; increment by 100ms *
retry^2 on each try
+ // -- fit here nicely so just do something simple; increment by
rsRpcRetryInterval millis *
+ // retry^2 on each try
// up to max of 10 seconds (don't want to back off too much in case of
situation change).
submitTask(this,
-Math.min(100 * (this.numberOfAttemptsSoFar *
this.numberOfAttemptsSoFar), 10 * 1000),
+Math.min(rsRpcRetryInterval * (this.numberOfAttemptsSoFar *
this.numberOfAttemptsSoFar),
+ 10 * 1000),
TimeUnit.MILLISECONDS);
return true;
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java
index dd1c085..0e1c280 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java
@@ -22,7 +22,8 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.io.StringWriter;
-
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
@@ -38,6 +39,7 @@ import org.apache.hadoop.hbase.master.ServerManager;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
+import org.apache.hadoop.hbase.util.Threads;
import org.apache.log4j.Appender;
import org.apache.log4j.Layout;
import org.apache.log4j.PatternLayout;
@@ -222,6 +224,40 @@ public class TestRegionServerReportForDuty {
[hbase] branch branch-2.2 updated (1a4b474 -> 64b5920)
This is an automated email from the ASF dual-hosted git repository. wchevreuil pushed a change to branch branch-2.2 in repository https://gitbox.apache.org/repos/asf/hbase.git. from 1a4b474 HBASE-24511 Ability to configure timeout between RPC retry to RS from master (#1861) add 64b5920 HBASE-24524 SyncTable logging improvements (#1878) No new revisions were added by this update. Summary of changes: .../java/org/apache/hadoop/hbase/mapreduce/SyncTable.java| 12 ++-- 1 file changed, 6 insertions(+), 6 deletions(-)
[hbase] branch branch-2 updated: HBASE-24511 Ability to configure timeout between RPC retry to RS from master (#1861)
This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new c025540 HBASE-24511 Ability to configure timeout between RPC retry to
RS from master (#1861)
c025540 is described below
commit c02554024fcae67d5c49ea215e6d322234037bc3
Author: sguggilam
AuthorDate: Wed Jun 10 00:23:08 2020 -0700
HBASE-24511 Ability to configure timeout between RPC retry to RS from
master (#1861)
Signed-off-by: Viraj Jasani
---
.../master/procedure/RSProcedureDispatcher.java| 17 +++---
.../TestRegionServerReportForDuty.java | 38 +-
2 files changed, 50 insertions(+), 5 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
index 74e00f5..3b4ee4c 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
@@ -235,12 +235,19 @@ public class RSProcedureDispatcher
private int numberOfAttemptsSoFar = 0;
private long maxWaitTime = -1;
+private final long rsRpcRetryInterval;
+private static final String RS_RPC_RETRY_INTERVAL_CONF_KEY =
+"hbase.regionserver.rpc.retry.interval";
+private static final int DEFAULT_RS_RPC_RETRY_INTERVAL = 100;
+
private ExecuteProceduresRequest.Builder request = null;
public ExecuteProceduresRemoteCall(final ServerName serverName,
final Set remoteProcedures) {
this.serverName = serverName;
this.remoteProcedures = remoteProcedures;
+ this.rsRpcRetryInterval =
master.getConfiguration().getLong(RS_RPC_RETRY_INTERVAL_CONF_KEY,
+DEFAULT_RS_RPC_RETRY_INTERVAL);
}
private AdminService.BlockingInterface getRsAdmin() throws IOException {
@@ -265,8 +272,8 @@ public class RSProcedureDispatcher
LOG.warn("Waiting a little before retrying {}, try={}, can wait up
to {}ms",
serverName, numberOfAttemptsSoFar, remainingTime);
numberOfAttemptsSoFar++;
- // Retry every 100ms up to maximum wait time.
- submitTask(this, 100, TimeUnit.MILLISECONDS);
+ // Retry every rsRpcRetryInterval millis up to maximum wait time.
+ submitTask(this, rsRpcRetryInterval, TimeUnit.MILLISECONDS);
return true;
}
LOG.warn("{} is throwing ServerNotRunningYetException for {}ms; trying
another server",
@@ -311,10 +318,12 @@ public class RSProcedureDispatcher
numberOfAttemptsSoFar++;
// Add some backoff here as the attempts rise otherwise if a stuck
condition, will fill logs
// with failed attempts. None of our backoff classes -- RetryCounter or
ClientBackoffPolicy
- // -- fit here nicely so just do something simple; increment by 100ms *
retry^2 on each try
+ // -- fit here nicely so just do something simple; increment by
rsRpcRetryInterval millis *
+ // retry^2 on each try
// up to max of 10 seconds (don't want to back off too much in case of
situation change).
submitTask(this,
-Math.min(100 * (this.numberOfAttemptsSoFar *
this.numberOfAttemptsSoFar), 10 * 1000),
+Math.min(rsRpcRetryInterval * (this.numberOfAttemptsSoFar *
this.numberOfAttemptsSoFar),
+ 10 * 1000),
TimeUnit.MILLISECONDS);
return true;
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java
index dd1c085..0e1c280 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java
@@ -22,7 +22,8 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.io.StringWriter;
-
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
@@ -38,6 +39,7 @@ import org.apache.hadoop.hbase.master.ServerManager;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
+import org.apache.hadoop.hbase.util.Threads;
import org.apache.log4j.Appender;
import org.apache.log4j.Layout;
import org.apache.log4j.PatternLayout;
@@ -222,6 +224,40 @@ public class TestRegionServerReportForDuty {
[hbase] branch branch-2.3 updated: HBASE-24524 SyncTable logging improvements (#1878)
This is an automated email from the ASF dual-hosted git repository.
wchevreuil pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.3 by this push:
new a1a58ae HBASE-24524 SyncTable logging improvements (#1878)
a1a58ae is described below
commit a1a58aef383995b280016a2ba44233bcef3a5a95
Author: Wellington Ramos Chevreuil
AuthorDate: Wed Jun 10 09:36:30 2020 +0100
HBASE-24524 SyncTable logging improvements (#1878)
Signed-off-by: Jan Hentschel
Signed-off-by: Viraj Jasani
(cherry picked from commit d8247ebae347915c3fbc73f8ab6c39b0984826b0)
---
.../java/org/apache/hadoop/hbase/mapreduce/SyncTable.java| 12 ++--
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/SyncTable.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/SyncTable.java
index cf9241d..6722501 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/SyncTable.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/SyncTable.java
@@ -363,16 +363,16 @@ public class SyncTable extends Configured implements Tool
{
boolean rowMatched;
int rowComparison = compareRowKeys(nextSourceRow, nextTargetRow);
if (rowComparison < 0) {
- if (LOG.isInfoEnabled()) {
-LOG.info("Target missing row: " + Bytes.toHex(nextSourceRow));
+ if (LOG.isDebugEnabled()) {
+LOG.debug("Target missing row: " + Bytes.toString(nextSourceRow));
}
context.getCounter(Counter.TARGETMISSINGROWS).increment(1);
rowMatched = syncRowCells(context, nextSourceRow, sourceCells,
EMPTY_CELL_SCANNER);
nextSourceRow = sourceCells.nextRow(); // advance only source to
next row
} else if (rowComparison > 0) {
- if (LOG.isInfoEnabled()) {
-LOG.info("Source missing row: " + Bytes.toHex(nextTargetRow));
+ if (LOG.isDebugEnabled()) {
+LOG.debug("Source missing row: " + Bytes.toString(nextTargetRow));
}
context.getCounter(Counter.SOURCEMISSINGROWS).increment(1);
@@ -557,10 +557,10 @@ public class SyncTable extends Configured implements Tool
{
if (LOG.isDebugEnabled()) {
LOG.debug("Different values: ");
LOG.debug(" source cell: " + sourceCell
- + " value: " + Bytes.toHex(sourceCell.getValueArray(),
+ + " value: " + Bytes.toString(sourceCell.getValueArray(),
sourceCell.getValueOffset(),
sourceCell.getValueLength()));
LOG.debug(" target cell: " + targetCell
- + " value: " + Bytes.toHex(targetCell.getValueArray(),
+ + " value: " + Bytes.toString(targetCell.getValueArray(),
targetCell.getValueOffset(),
targetCell.getValueLength()));
}
context.getCounter(Counter.DIFFERENTCELLVALUES).increment(1);
[hbase] branch branch-2.2 updated: HBASE-24511 Ability to configure timeout between RPC retry to RS from master (#1861)
This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch branch-2.2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.2 by this push:
new 1a4b474 HBASE-24511 Ability to configure timeout between RPC retry to
RS from master (#1861)
1a4b474 is described below
commit 1a4b474c2b057bdcc48a4bb2eeb17ca77b111bc8
Author: sguggilam
AuthorDate: Wed Jun 10 00:23:08 2020 -0700
HBASE-24511 Ability to configure timeout between RPC retry to RS from
master (#1861)
Signed-off-by: Viraj Jasani
---
.../master/procedure/RSProcedureDispatcher.java| 20 ++--
.../TestRegionServerReportForDuty.java | 38 +-
2 files changed, 55 insertions(+), 3 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
index b469cb8..ae60848 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
@@ -211,12 +211,19 @@ public class RSProcedureDispatcher
private int numberOfAttemptsSoFar = 0;
private long maxWaitTime = -1;
+private final long rsRpcRetryInterval;
+private static final String RS_RPC_RETRY_INTERVAL_CONF_KEY =
+"hbase.regionserver.rpc.retry.interval";
+private static final int DEFAULT_RS_RPC_RETRY_INTERVAL = 100;
+
private ExecuteProceduresRequest.Builder request = null;
public ExecuteProceduresRemoteCall(final ServerName serverName,
final Set remoteProcedures) {
this.serverName = serverName;
this.remoteProcedures = remoteProcedures;
+ this.rsRpcRetryInterval =
master.getConfiguration().getLong(RS_RPC_RETRY_INTERVAL_CONF_KEY,
+DEFAULT_RS_RPC_RETRY_INTERVAL);
}
private AdminService.BlockingInterface getRsAdmin() throws IOException {
@@ -241,7 +248,8 @@ public class RSProcedureDispatcher
LOG.warn("waiting a little before trying on the same server={}," +
" try={}, can wait up to {}ms", serverName, numberOfAttemptsSoFar,
remainingTime);
numberOfAttemptsSoFar++;
- submitTask(this, 100, TimeUnit.MILLISECONDS);
+ // Retry every rsRpcRetryInterval millis up to maximum wait time.
+ submitTask(this, rsRpcRetryInterval, TimeUnit.MILLISECONDS);
return true;
}
LOG.warn("server {} is not up for a while; try a new one", serverName);
@@ -283,7 +291,15 @@ public class RSProcedureDispatcher
e.toString(), numberOfAttemptsSoFar);
}
numberOfAttemptsSoFar++;
- submitTask(this, 100, TimeUnit.MILLISECONDS);
+ // Add some backoff here as the attempts rise otherwise if a stuck
condition, will fill logs
+ // with failed attempts. None of our backoff classes -- RetryCounter or
ClientBackoffPolicy
+ // -- fit here nicely so just do something simple; increment by
rsRpcRetryInterval millis *
+ // retry^2 on each try
+ // up to max of 10 seconds (don't want to back off too much in case of
situation change).
+ submitTask(this,
+Math.min(rsRpcRetryInterval * (this.numberOfAttemptsSoFar *
this.numberOfAttemptsSoFar),
+ 10 * 1000),
+TimeUnit.MILLISECONDS);
return true;
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java
index aaf2d2e..2821e36 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java
@@ -22,7 +22,8 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.io.StringWriter;
-
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
@@ -38,6 +39,7 @@ import org.apache.hadoop.hbase.master.ServerManager;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
+import org.apache.hadoop.hbase.util.Threads;
import org.apache.log4j.Appender;
import org.apache.log4j.Layout;
import org.apache.log4j.PatternLayout;
@@ -221,6 +223,40 @@ public class TestRegionServerReportForDuty {
tablesOnMaster? 3: 2);
}
+
+ /**
+ * Tests region sever reportForDuty with RS RPC retry
+ */
+ @Test
+ public void
[hbase] branch master updated: HBASE-24524 SyncTable logging improvements (#1878)
This is an automated email from the ASF dual-hosted git repository.
wchevreuil pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new d8247eb HBASE-24524 SyncTable logging improvements (#1878)
d8247eb is described below
commit d8247ebae347915c3fbc73f8ab6c39b0984826b0
Author: Wellington Ramos Chevreuil
AuthorDate: Wed Jun 10 09:36:30 2020 +0100
HBASE-24524 SyncTable logging improvements (#1878)
Signed-off-by: Jan Hentschel
Signed-off-by: Viraj Jasani
---
.../java/org/apache/hadoop/hbase/mapreduce/SyncTable.java| 12 ++--
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/SyncTable.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/SyncTable.java
index bb20061..b60f828 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/SyncTable.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/SyncTable.java
@@ -364,16 +364,16 @@ public class SyncTable extends Configured implements Tool
{
boolean rowMatched;
int rowComparison = compareRowKeys(nextSourceRow, nextTargetRow);
if (rowComparison < 0) {
- if (LOG.isInfoEnabled()) {
-LOG.info("Target missing row: " + Bytes.toHex(nextSourceRow));
+ if (LOG.isDebugEnabled()) {
+LOG.debug("Target missing row: " + Bytes.toString(nextSourceRow));
}
context.getCounter(Counter.TARGETMISSINGROWS).increment(1);
rowMatched = syncRowCells(context, nextSourceRow, sourceCells,
EMPTY_CELL_SCANNER);
nextSourceRow = sourceCells.nextRow(); // advance only source to
next row
} else if (rowComparison > 0) {
- if (LOG.isInfoEnabled()) {
-LOG.info("Source missing row: " + Bytes.toHex(nextTargetRow));
+ if (LOG.isDebugEnabled()) {
+LOG.debug("Source missing row: " + Bytes.toString(nextTargetRow));
}
context.getCounter(Counter.SOURCEMISSINGROWS).increment(1);
@@ -558,10 +558,10 @@ public class SyncTable extends Configured implements Tool
{
if (LOG.isDebugEnabled()) {
LOG.debug("Different values: ");
LOG.debug(" source cell: " + sourceCell
- + " value: " + Bytes.toHex(sourceCell.getValueArray(),
+ + " value: " + Bytes.toString(sourceCell.getValueArray(),
sourceCell.getValueOffset(),
sourceCell.getValueLength()));
LOG.debug(" target cell: " + targetCell
- + " value: " + Bytes.toHex(targetCell.getValueArray(),
+ + " value: " + Bytes.toString(targetCell.getValueArray(),
targetCell.getValueOffset(),
targetCell.getValueLength()));
}
context.getCounter(Counter.DIFFERENTCELLVALUES).increment(1);
[hbase-operator-tools] branch master updated: HBASE-23927 recommit the fix with updates for PR review feedback (#64)
This is an automated email from the ASF dual-hosted git repository.
huaxiangsun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase-operator-tools.git
The following commit(s) were added to refs/heads/master by this push:
new edf48e7 HBASE-23927 recommit the fix with updates for PR review
feedback (#64)
edf48e7 is described below
commit edf48e71fd3cd6a88d29a9f6cf95ebe83d963085
Author: clarax
AuthorDate: Wed Jun 10 16:37:30 2020 -0700
HBASE-23927 recommit the fix with updates for PR review feedback (#64)
HBASE-23927 hbck2 assigns command should accept a file containing a list of
region names
Signed-off-by: Nick Dimiduk
Signed-off-by: Huaxiang Sun
---
hbase-hbck2/README.md | 9 ++-
.../src/main/java/org/apache/hbase/HBCK2.java | 37 +++---
.../src/test/java/org/apache/hbase/TestHBCK2.java | 80 +++---
3 files changed, 92 insertions(+), 34 deletions(-)
diff --git a/hbase-hbck2/README.md b/hbase-hbck2/README.md
index 39f3711..e7c01ff 100644
--- a/hbase-hbck2/README.md
+++ b/hbase-hbck2/README.md
@@ -124,17 +124,20 @@ Command:
SEE ALSO: reportMissingRegionsInMeta
SEE ALSO: fixMeta
- assigns [OPTIONS] ...
+ assigns [OPTIONS] ...
Options:
-o,--override override ownership by another procedure
+-i,--inputFiles take one or more encoded region names
A 'raw' assign that can be used even during Master initialization (if
the -skip flag is specified). Skirts Coprocessors. Pass one or more
encoded region names. 1588230740 is the hard-coded name for the
hbase:meta region and de00010733901a05f5a2a3a382e27dd4 is an example of
what a user-space encoded region name looks like. For example:
- $ HBCK2 assign 1588230740 de00010733901a05f5a2a3a382e27dd4
+ $ HBCK2 assigns 1588230740 de00010733901a05f5a2a3a382e27dd4
Returns the pid(s) of the created AssignProcedure(s) or -1 if none.
-
+ If -i or --inputFiles is specified, pass one or more input file names.
+ Each file contains encoded region names, one per line. For example:
+ $ HBCK2 assigns -i fileName1 fileName2
bypass [OPTIONS] ...
Options:
-o,--override override if procedure is running/stuck
diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
index 46de2ec..4ab1b4b 100644
--- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
+++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
@@ -17,10 +17,7 @@
*/
package org.apache.hbase;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintWriter;
-import java.io.StringWriter;
+import java.io.*;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Arrays;
@@ -35,6 +32,9 @@ import java.util.concurrent.Future;
import java.util.function.Function;
import java.util.stream.Collectors;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.LineIterator;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
@@ -109,7 +109,8 @@ public class HBCK2 extends Configured implements
org.apache.hadoop.util.Tool {
private static final String ADD_MISSING_REGIONS_IN_META_FOR_TABLES =
"addFsRegionsMissingInMeta";
private static final String REPORT_MISSING_REGIONS_IN_META =
"reportMissingRegionsInMeta";
- static final String EXTRA_REGIONS_IN_META = "extraRegionsInMeta";
+ private static final String EXTRA_REGIONS_IN_META = "extraRegionsInMeta";
+
private Configuration conf;
static final String [] MINIMUM_HBCK2_VERSION = {"2.0.3", "2.1.1", "2.2.0",
"3.0.0"};
private boolean skipCheck = false;
@@ -280,10 +281,12 @@ public class HBCK2 extends Configured implements
org.apache.hadoop.util.Tool {
}
}
- List assigns(Hbck hbck, String [] args) throws IOException {
+ List assigns(Hbck hbck, String[] args) throws IOException {
Options options = new Options();
Option override = Option.builder("o").longOpt("override").build();
+Option inputFile = Option.builder("i").longOpt("inputFiles").build();
options.addOption(override);
+options.addOption(inputFile);
// Parse command-line.
CommandLineParser parser = new DefaultParser();
CommandLine commandLine;
@@ -294,7 +297,21 @@ public class HBCK2 extends Configured implements
org.apache.hadoop.util.Tool {
return null;
}
boolean overrideFlag = commandLine.hasOption(override.getOpt());
-return hbck.assigns(commandLine.getArgList(), overrideFlag);
+
+List argList = commandLine.getArgList();
+if (!commandLine.hasOption(inputFile.getOpt())) {
+ return hbck.assigns(argList, overrideFlag);
+}
+List assignmentList = new ArrayList<>();
+for (String filePath : argList) {
+ try (InputStream fileStream = new FileInputStream(filePath)){
+LineIterator it =
[hbase] branch master updated: HBASE-8458 Support for batch version of checkAndMutate()
This is an automated email from the ASF dual-hosted git repository.
elserj pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new efd5a96 HBASE-8458 Support for batch version of checkAndMutate()
efd5a96 is described below
commit efd5a962e5a6aa07dcba4b55f8b165ea3dbbd6ef
Author: Toshihiro Suzuki
AuthorDate: Mon May 4 16:53:41 2020 +0900
HBASE-8458 Support for batch version of checkAndMutate()
Closes #1648
Signed-off-by: Josh Elser
---
.../hbase/client/AsyncBatchRpcRetryingCaller.java | 16 +-
.../org/apache/hadoop/hbase/client/AsyncTable.java | 47 ++
.../apache/hadoop/hbase/client/AsyncTableImpl.java | 11 +
.../apache/hadoop/hbase/client/CheckAndMutate.java | 362 ++
.../org/apache/hadoop/hbase/client/Mutation.java | 20 +-
.../hadoop/hbase/client/RawAsyncTableImpl.java | 62 +-
.../java/org/apache/hadoop/hbase/client/Table.java | 41 ++
.../hadoop/hbase/client/TableOverAsyncTable.java | 10 +
.../hbase/shaded/protobuf/RequestConverter.java| 215 +++---
.../hbase/shaded/protobuf/ResponseConverter.java | 21 +-
.../src/main/protobuf/client/Client.proto | 11 +-
.../hadoop/hbase/rest/client/RemoteHTable.java | 11 +
.../hadoop/hbase/regionserver/RSRpcServices.java | 262 ++--
.../hadoop/hbase/client/DummyAsyncTable.java | 10 +
.../apache/hadoop/hbase/client/TestAsyncTable.java | 732 -
.../hadoop/hbase/client/TestAsyncTableBatch.java | 54 ++
.../hadoop/hbase/client/TestCheckAndMutate.java| 574 +++-
.../hadoop/hbase/client/TestFromClientSide3.java | 54 ++
.../hbase/client/TestMalformedCellFromClient.java | 5 +-
.../hadoop/hbase/thrift2/client/ThriftTable.java | 11 +
20 files changed, 2270 insertions(+), 259 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncBatchRpcRetryingCaller.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncBatchRpcRetryingCaller.java
index 464eff5..7e05b05 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncBatchRpcRetryingCaller.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncBatchRpcRetryingCaller.java
@@ -256,7 +256,7 @@ class AsyncBatchRpcRetryingCaller {
}
private ClientProtos.MultiRequest buildReq(Map
actionsByRegion,
- List cells, Map rowMutationsIndexMap)
throws IOException {
+ List cells, Map indexMap) throws
IOException {
ClientProtos.MultiRequest.Builder multiRequestBuilder =
ClientProtos.MultiRequest.newBuilder();
ClientProtos.RegionAction.Builder regionActionBuilder =
ClientProtos.RegionAction.newBuilder();
ClientProtos.Action.Builder actionBuilder =
ClientProtos.Action.newBuilder();
@@ -264,14 +264,14 @@ class AsyncBatchRpcRetryingCaller {
for (Map.Entry entry : actionsByRegion.entrySet()) {
long nonceGroup = conn.getNonceGenerator().getNonceGroup();
// multiRequestBuilder will be populated with region actions.
- // rowMutationsIndexMap will be non-empty after the call if there is
RowMutations in the
+ // indexMap will be non-empty after the call if there is
RowMutations/CheckAndMutate in the
// action list.
RequestConverter.buildNoDataRegionActions(entry.getKey(),
entry.getValue().actions.stream()
.sorted((a1, a2) -> Integer.compare(a1.getOriginalIndex(),
a2.getOriginalIndex()))
.collect(Collectors.toList()),
-cells, multiRequestBuilder, regionActionBuilder, actionBuilder,
mutationBuilder, nonceGroup,
-rowMutationsIndexMap);
+cells, multiRequestBuilder, regionActionBuilder, actionBuilder,
mutationBuilder,
+nonceGroup, indexMap);
}
return multiRequestBuilder.build();
}
@@ -367,10 +367,10 @@ class AsyncBatchRpcRetryingCaller {
List cells = new ArrayList<>();
// Map from a created RegionAction to the original index for a
RowMutations within
// the original list of actions. This will be used to process the results
when there
-// is RowMutations in the action list.
-Map rowMutationsIndexMap = new HashMap<>();
+// is RowMutations/CheckAndMutate in the action list.
+Map indexMap = new HashMap<>();
try {
- req = buildReq(serverReq.actionsByRegion, cells, rowMutationsIndexMap);
+ req = buildReq(serverReq.actionsByRegion, cells, indexMap);
} catch (IOException e) {
onError(serverReq.actionsByRegion, tries, e, serverName);
return;
@@ -387,7 +387,7 @@ class AsyncBatchRpcRetryingCaller {
} else {
try {
onComplete(serverReq.actionsByRegion, tries, serverName,
ResponseConverter.getResults(req,
-rowMutationsIndexMap, resp, controller.cellScanner()));
+indexMap, resp, controller.cellScanner()));
} catch (Exception e) {
