[hbase] branch HBASE-21512 updated (b2dc796 -> 19c9908)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a change to branch HBASE-21512
in repository https://gitbox.apache.org/repos/asf/hbase.git.
discard b2dc796 HBASE-21538 Rewrite RegionReplicaFlushHandler to use
AsyncClusterConnection
discard fd3e534 HBASE-21579 Use AsyncClusterConnection for
HBaseInterClusterReplicationEndpoint
discard fa8c63a HBASE-21526 Use AsyncClusterConnection in ServerManager for
getRsAdmin
discard 53a9645 HBASE-21516 Use AsyncConnection instead of Connection in
SecureBulkLoadManager
discard 009a31b HBASE-21515 Also initialize an AsyncClusterConnection in
HRegionServer
add fbf7937 HBASE-21663 Add replica scan support
new f9914c5 HBASE-21515 Also initialize an AsyncClusterConnection in
HRegionServer
new ebf6f9c HBASE-21516 Use AsyncConnection instead of Connection in
SecureBulkLoadManager
new b542ca8 HBASE-21526 Use AsyncClusterConnection in ServerManager for
getRsAdmin
new 314094b HBASE-21579 Use AsyncClusterConnection for
HBaseInterClusterReplicationEndpoint
new 19c9908 HBASE-21538 Rewrite RegionReplicaFlushHandler to use
AsyncClusterConnection
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (b2dc796)
\
N -- N -- N refs/heads/HBASE-21512 (19c9908)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../client/AsyncAdminRequestRetryingCaller.java| 15 +-
.../hbase/client/AsyncBatchRpcRetryingCaller.java | 6 +-
.../hadoop/hbase/client/AsyncClientScanner.java| 103 +++-
.../hbase/client/AsyncConnectionConfiguration.java | 21 +++
.../hadoop/hbase/client/AsyncConnectionImpl.java | 5 +-
.../AsyncMasterRequestRpcRetryingCaller.java | 16 +-
.../hbase/client/AsyncRpcRetryingCaller.java | 15 +-
.../client/AsyncRpcRetryingCallerFactory.java | 6 +-
.../AsyncScanSingleRegionRpcRetryingCaller.java| 9 +-
.../AsyncServerRequestRpcRetryingCaller.java | 16 +-
.../AsyncSingleRequestRpcRetryingCaller.java | 10 +-
.../hbase/client/ConnectionConfiguration.java | 9 +-
.../hadoop/hbase/client/ConnectionUtils.java | 101 ++-
.../hadoop/hbase/client/RawAsyncTableImpl.java | 93 ++
... AbstractTestAsyncTableRegionReplicasRead.java} | 102 +--
.../client/TestAsyncTableRegionReplicasGet.java| 187 +
.../client/TestAsyncTableRegionReplicasScan.java | 76 +
17 files changed, 370 insertions(+), 420 deletions(-)
copy
hbase-server/src/test/java/org/apache/hadoop/hbase/client/{TestAsyncTableRegionReplicasGet.java
=> AbstractTestAsyncTableRegionReplicasRead.java} (72%)
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableRegionReplicasScan.java
[hbase] 02/05: HBASE-21516 Use AsyncConnection instead of Connection in SecureBulkLoadManager
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-21512
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit ebf6f9cc013a418f65a69ec472de1522aa1d33ff
Author: zhangduo
AuthorDate: Sat Dec 1 21:15:48 2018 +0800
HBASE-21516 Use AsyncConnection instead of Connection in
SecureBulkLoadManager
---
.../apache/hadoop/hbase/protobuf/ProtobufUtil.java | 5 +-
.../hadoop/hbase/shaded/protobuf/ProtobufUtil.java | 7 ++-
.../hadoop/hbase/regionserver/HRegionServer.java | 2 +-
.../hbase/regionserver/SecureBulkLoadManager.java | 24 -
.../hadoop/hbase/security/token/TokenUtil.java | 57 +-
.../hadoop/hbase/security/token/TestTokenUtil.java | 42
6 files changed, 96 insertions(+), 41 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index a3d49b5..d9e620b 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -261,13 +261,12 @@ public final class ProtobufUtil {
* just {@link ServiceException}. Prefer this method to
* {@link #getRemoteException(ServiceException)} because trying to
* contain direct protobuf references.
- * @param e
*/
- public static IOException handleRemoteException(Exception e) {
+ public static IOException handleRemoteException(Throwable e) {
return makeIOExceptionOfException(e);
}
- private static IOException makeIOExceptionOfException(Exception e) {
+ private static IOException makeIOExceptionOfException(Throwable e) {
Throwable t = e;
if (e instanceof ServiceException ||
e instanceof
org.apache.hbase.thirdparty.com.google.protobuf.ServiceException) {
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
index fea81f1..de2fb7d 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
@@ -40,7 +40,6 @@ import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ByteBufferExtendedCell;
@@ -123,6 +122,7 @@ import
org.apache.hbase.thirdparty.com.google.protobuf.Service;
import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
import org.apache.hbase.thirdparty.com.google.protobuf.TextFormat;
import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
+
import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
@@ -343,13 +343,12 @@ public final class ProtobufUtil {
* just {@link ServiceException}. Prefer this method to
* {@link #getRemoteException(ServiceException)} because trying to
* contain direct protobuf references.
- * @param e
*/
- public static IOException handleRemoteException(Exception e) {
+ public static IOException handleRemoteException(Throwable e) {
return makeIOExceptionOfException(e);
}
- private static IOException makeIOExceptionOfException(Exception e) {
+ private static IOException makeIOExceptionOfException(Throwable e) {
Throwable t = e;
if (e instanceof ServiceException) {
t = e.getCause();
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index b4b1d3e..bbc9d3c 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1937,7 +1937,7 @@ public class HRegionServer extends HasThread implements
if (!isStopped() && !isAborted()) {
initializeThreads();
}
-this.secureBulkLoadManager = new SecureBulkLoadManager(this.conf,
clusterConnection);
+this.secureBulkLoadManager = new SecureBulkLoadManager(this.conf,
asyncClusterConnection);
this.secureBulkLoadManager.start();
// Health checker thread.
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java
index 566a6b6..add6519 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBul
[hbase] 04/05: HBASE-21579 Use AsyncClusterConnection for HBaseInterClusterReplicationEndpoint
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-21512
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 314094b9c32786094bf6972cd849afc587ffdd6b
Author: zhangduo
AuthorDate: Tue Jan 1 21:27:14 2019 +0800
HBASE-21579 Use AsyncClusterConnection for
HBaseInterClusterReplicationEndpoint
---
.../hbase/client/AsyncRegionServerAdmin.java | 14 +---
.../hbase/protobuf/ReplicationProtbufUtil.java | 35 ++-
.../HBaseInterClusterReplicationEndpoint.java | 31 +
.../regionserver/ReplicationSinkManager.java | 40 --
.../hbase/replication/SyncReplicationTestBase.java | 12 +++
.../regionserver/TestReplicationSinkManager.java | 21 +---
6 files changed, 74 insertions(+), 79 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java
index 9accd89..b9141a9 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.client;
import java.io.IOException;
import java.util.concurrent.CompletableFuture;
+import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.ipc.HBaseRpcController;
import org.apache.yetus.audience.InterfaceAudience;
@@ -94,9 +95,9 @@ public class AsyncRegionServerAdmin {
void call(AdminService.Interface stub, HBaseRpcController controller,
RpcCallback done);
}
- private CompletableFuture call(RpcCall rpcCall) {
+ private CompletableFuture call(RpcCall rpcCall,
CellScanner cellScanner) {
CompletableFuture future = new CompletableFuture<>();
-HBaseRpcController controller = conn.rpcControllerFactory.newController();
+HBaseRpcController controller =
conn.rpcControllerFactory.newController(cellScanner);
try {
rpcCall.call(conn.getAdminStub(server), controller, new
RpcCallback() {
@@ -115,6 +116,10 @@ public class AsyncRegionServerAdmin {
return future;
}
+ private CompletableFuture call(RpcCall rpcCall) {
+return call(rpcCall, null);
+ }
+
public CompletableFuture
getRegionInfo(GetRegionInfoRequest request) {
return call((stub, controller, done) -> stub.getRegionInfo(controller,
request, done));
}
@@ -154,8 +159,9 @@ public class AsyncRegionServerAdmin {
}
public CompletableFuture replicateWALEntry(
- ReplicateWALEntryRequest request) {
-return call((stub, controller, done) -> stub.replicateWALEntry(controller,
request, done));
+ ReplicateWALEntryRequest request, CellScanner cellScanner) {
+return call((stub, controller, done) -> stub.replicateWALEntry(controller,
request, done),
+ cellScanner);
}
public CompletableFuture
replay(ReplicateWALEntryRequest request) {
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java
index c1b3911..74fad26 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java
@@ -20,51 +20,54 @@ package org.apache.hadoop.hbase.protobuf;
import java.io.IOException;
+import java.io.InterruptedIOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import java.util.concurrent.ExecutionException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.PrivateCellUtil;
+import org.apache.hadoop.hbase.client.AsyncRegionServerAdmin;
import org.apache.hadoop.hbase.io.SizedCellScanner;
-import org.apache.hadoop.hbase.ipc.HBaseRpcController;
-import org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl;
import org.apache.hadoop.hbase.regionserver.wal.WALCellCodec;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.wal.WAL.Entry;
import org.apache.hadoop.hbase.wal.WALEdit;
import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.hbase.thirdparty.com.google.common.base.Throwables;
import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
-import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;
@InterfaceAudience.Private
public class ReplicationProtbufUtil {
+
/**
- * A helper to replicate a list of WAL entries using admin protocol.
- * @param admin Admin service
+ * A helper to r
[hbase] 03/05: HBASE-21526 Use AsyncClusterConnection in ServerManager for getRsAdmin
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-21512
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit b542ca810680409abda91c733e2d3fed91e4c622
Author: zhangduo
AuthorDate: Thu Dec 6 21:25:34 2018 +0800
HBASE-21526 Use AsyncClusterConnection in ServerManager for getRsAdmin
---
.../hbase/client/AsyncClusterConnection.java | 6 +
.../hadoop/hbase/client/AsyncConnectionImpl.java | 4 +
.../hbase/client/AsyncRegionServerAdmin.java | 210 +
.../org/apache/hadoop/hbase/util/FutureUtils.java | 2 +-
.../org/apache/hadoop/hbase/master/HMaster.java| 15 +-
.../apache/hadoop/hbase/master/ServerManager.java | 67 ---
.../master/procedure/RSProcedureDispatcher.java| 44 +++--
7 files changed, 262 insertions(+), 86 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
index c7dea25..1327fd7 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.client;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.ipc.RpcClient;
import org.apache.yetus.audience.InterfaceAudience;
@@ -27,6 +28,11 @@ import org.apache.yetus.audience.InterfaceAudience;
public interface AsyncClusterConnection extends AsyncConnection {
/**
+ * Get the admin service for the given region server.
+ */
+ AsyncRegionServerAdmin getRegionServerAdmin(ServerName serverName);
+
+ /**
* Get the nonce generator for this connection.
*/
NonceGenerator getNonceGenerator();
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
index 50e27c4..9bead83 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
@@ -359,4 +359,8 @@ class AsyncConnectionImpl implements AsyncClusterConnection
{
return new HBaseHbck(MasterProtos.HbckService.newBlockingStub(
rpcClient.createBlockingRpcChannel(masterServer, user, rpcTimeout)),
rpcControllerFactory);
}
+
+ public AsyncRegionServerAdmin getRegionServerAdmin(ServerName serverName) {
+return new AsyncRegionServerAdmin(serverName, this);
+ }
}
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java
new file mode 100644
index 000..9accd89
--- /dev/null
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java
@@ -0,0 +1,210 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import java.io.IOException;
+import java.util.concurrent.CompletableFuture;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.ipc.HBaseRpcController;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback;
+import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
+
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesRequest;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesResponse;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheRequest;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheResponse;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionResponse;
+import
org.apache.hadoop.hbase.shaded.protobuf.ge
[hbase] 01/05: HBASE-21515 Also initialize an AsyncClusterConnection in HRegionServer
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-21512
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit f9914c55419c97eac4616253e7fb5b90a8e3ab8f
Author: zhangduo
AuthorDate: Fri Nov 30 08:23:47 2018 +0800
HBASE-21515 Also initialize an AsyncClusterConnection in HRegionServer
---
.../hbase/client/AsyncClusterConnection.java | 38 +
.../hadoop/hbase/client/AsyncConnectionImpl.java | 14 +++--
.../hbase/client/ClusterConnectionFactory.java | 63 ++
.../hadoop/hbase/client/ConnectionFactory.java | 5 +-
.../apache/hadoop/hbase/util/ReflectionUtils.java | 22
.../main/java/org/apache/hadoop/hbase/Server.java | 20 +++
.../org/apache/hadoop/hbase/master/HMaster.java| 3 ++
.../hadoop/hbase/regionserver/HRegionServer.java | 56 +--
.../regionserver/ReplicationSyncUp.java| 6 +++
.../hadoop/hbase/MockRegionServerServices.java | 5 ++
.../client/TestAsyncNonMetaRegionLocator.java | 2 +-
...stAsyncNonMetaRegionLocatorConcurrenyLimit.java | 2 +-
.../client/TestAsyncRegionLocatorTimeout.java | 2 +-
.../TestAsyncSingleRequestRpcRetryingCaller.java | 4 +-
.../hbase/client/TestAsyncTableNoncedRetry.java| 2 +-
.../hbase/master/MockNoopMasterServices.java | 6 +++
.../hadoop/hbase/master/MockRegionServer.java | 5 ++
.../hbase/master/TestActiveMasterManager.java | 6 +++
.../hbase/master/cleaner/TestHFileCleaner.java | 6 +++
.../hbase/master/cleaner/TestHFileLinkCleaner.java | 6 +++
.../hbase/master/cleaner/TestLogsCleaner.java | 6 +++
.../cleaner/TestReplicationHFileCleaner.java | 6 +++
.../hbase/regionserver/TestHeapMemoryManager.java | 6 +++
.../hbase/regionserver/TestSplitLogWorker.java | 6 +++
.../hadoop/hbase/regionserver/TestWALLockup.java | 6 +++
.../replication/TestReplicationTrackerZKImpl.java | 6 +++
.../regionserver/TestReplicationSourceManager.java | 6 +++
.../security/token/TestTokenAuthentication.java| 6 +++
.../org/apache/hadoop/hbase/util/MockServer.java | 6 +++
29 files changed, 290 insertions(+), 37 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
new file mode 100644
index 000..c7dea25
--- /dev/null
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import org.apache.hadoop.hbase.ipc.RpcClient;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * The asynchronous connection for internal usage.
+ */
+@InterfaceAudience.Private
+public interface AsyncClusterConnection extends AsyncConnection {
+
+ /**
+ * Get the nonce generator for this connection.
+ */
+ NonceGenerator getNonceGenerator();
+
+ /**
+ * Get the rpc client we used to communicate with other servers.
+ */
+ RpcClient getRpcClient();
+}
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
index 3cbd950..50e27c4 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
@@ -23,6 +23,7 @@ import static
org.apache.hadoop.hbase.client.NonceGenerator.CLIENT_NONCES_ENABLE
import static org.apache.hadoop.hbase.util.FutureUtils.addListener;
import java.io.IOException;
+import java.net.SocketAddress;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -63,7 +64,7 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MasterServ
* The implementation of AsyncConnection.
*/
@InterfaceAudience.Private
-class AsyncConnectionImpl implements AsyncConnection {
+class AsyncConnectionImpl implements AsyncClusterConnecti
[hbase] 05/05: HBASE-21538 Rewrite RegionReplicaFlushHandler to use AsyncClusterConnection
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-21512
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 19c99087ed5742acabf7d649ef9fc558e9ff3619
Author: Duo Zhang
AuthorDate: Wed Dec 12 09:33:33 2018 +0800
HBASE-21538 Rewrite RegionReplicaFlushHandler to use AsyncClusterConnection
---
.../hbase/client/AsyncClusterConnection.java | 8 ++
.../hadoop/hbase/client/AsyncConnectionImpl.java | 8 ++
.../hbase/client/ClusterConnectionFactory.java | 16 +--
.../hadoop/hbase/client/RawAsyncHBaseAdmin.java| 36 ---
.../org/apache/hadoop/hbase/util/FutureUtils.java | 22 +
.../master/procedure/RSProcedureDispatcher.java| 34 +--
.../hbase/protobuf/ReplicationProtbufUtil.java | 15 +--
.../hadoop/hbase/regionserver/HRegionServer.java | 3 +-
.../handler/RegionReplicaFlushHandler.java | 110 -
9 files changed, 132 insertions(+), 120 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
index 1327fd7..f1f64ca 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
@@ -17,10 +17,13 @@
*/
package org.apache.hadoop.hbase.client;
+import java.util.concurrent.CompletableFuture;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.ipc.RpcClient;
import org.apache.yetus.audience.InterfaceAudience;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse;
+
/**
* The asynchronous connection for internal usage.
*/
@@ -41,4 +44,9 @@ public interface AsyncClusterConnection extends
AsyncConnection {
* Get the rpc client we used to communicate with other servers.
*/
RpcClient getRpcClient();
+
+ /**
+ * Flush a region and get the response.
+ */
+ CompletableFuture flush(byte[] regionName, boolean
writeFlushWALMarker);
}
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
index 9bead83..ce6bfac 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
@@ -55,6 +55,7 @@ import
org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer;
import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ClientService;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsMasterRunningResponse;
@@ -363,4 +364,11 @@ class AsyncConnectionImpl implements
AsyncClusterConnection {
public AsyncRegionServerAdmin getRegionServerAdmin(ServerName serverName) {
return new AsyncRegionServerAdmin(serverName, this);
}
+
+ @Override
+ public CompletableFuture flush(byte[] regionName,
+ boolean writeFlushWALMarker) {
+RawAsyncHBaseAdmin admin = (RawAsyncHBaseAdmin) getAdmin();
+return admin.flushRegionInternal(regionName, writeFlushWALMarker);
+ }
}
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterConnectionFactory.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterConnectionFactory.java
index 68c0630..79484db 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterConnectionFactory.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterConnectionFactory.java
@@ -18,15 +18,12 @@
package org.apache.hadoop.hbase.client;
import java.io.IOException;
-import java.io.InterruptedIOException;
import java.net.SocketAddress;
-import java.util.concurrent.ExecutionException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.util.FutureUtils;
import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hbase.thirdparty.com.google.common.base.Throwables;
-
/**
* The factory for creating {@link AsyncClusterConnection}.
*/
@@ -48,16 +45,7 @@ public final class ClusterConnectionFactory {
public static AsyncClusterConnection
createAsyncClusterConnection(Configuration conf,
SocketAddress localAddress, User user) throws IOException {
AsyncRegistry registry = AsyncRegistryFactory.getRegistry(conf);
-String clusterId;
-try {
- clusterId = registry.getClusterId().get();
-} catch (InterruptedException
[hbase-site] branch asf-site updated: this should hit pubsub and mailing list, finally!
This is an automated email from the ASF dual-hosted git repository. git-site-role pushed a commit to branch asf-site in repository https://gitbox.apache.org/repos/asf/hbase-site.git The following commit(s) were added to refs/heads/asf-site by this push: new 0abdb46 this should hit pubsub and mailing list, finally! 0abdb46 is described below commit 0abdb461aa43f028db81703af54bf554fcec0477 Author: jenkins AuthorDate: Fri Jan 11 09:22:19 2019 + this should hit pubsub and mailing list, finally! --- index.html | 2 -- 1 file changed, 2 deletions(-) diff --git a/index.html b/index.html index df9bbc4..ff755de 100644 --- a/index.html +++ b/index.html @@ -435,5 +435,3 @@ Apache HBase is an open-source, distributed, versioned, non-relational database - -
[hbase] branch HBASE-21512 updated: HBASE-21671 Rewrite RegionReplicaReplicationEndpoint to use AsyncClusterConnection
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-21512
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/HBASE-21512 by this push:
new d6fbe51 HBASE-21671 Rewrite RegionReplicaReplicationEndpoint to use
AsyncClusterConnection
d6fbe51 is described below
commit d6fbe513638fa66b578104825b77b047231ae31e
Author: Duo Zhang
AuthorDate: Fri Jan 11 16:22:24 2019 +0800
HBASE-21671 Rewrite RegionReplicaReplicationEndpoint to use
AsyncClusterConnection
---
.../hadoop/hbase/client/AsyncConnectionImpl.java | 24 +-
.../hbase/client/AsyncClusterConnection.java | 17 +
.../hbase/client/AsyncClusterConnectionImpl.java | 80 +++
.../AsyncRegionReplicaReplayRetryingCaller.java| 146
.../hbase/client/AsyncRegionServerAdmin.java | 5 +-
.../hbase/client/ClusterConnectionFactory.java | 2 +-
.../hbase/protobuf/ReplicationProtbufUtil.java | 31 +-
.../handler/RegionReplicaFlushHandler.java | 3 +-
.../hbase/replication/ReplicationEndpoint.java | 35 +-
.../RegionReplicaReplicationEndpoint.java | 783 +++--
.../regionserver/ReplicationSource.java| 2 +-
.../hbase/client/TestAsyncTableNoncedRetry.java| 2 +-
.../TestRegionReplicaReplicationEndpoint.java | 56 +-
...stRegionReplicaReplicationEndpointNoMaster.java | 99 ++-
14 files changed, 627 insertions(+), 658 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
index ce6bfac..f6a2149 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
@@ -55,7 +55,6 @@ import
org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer;
import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
-import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ClientService;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsMasterRunningResponse;
@@ -65,7 +64,7 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MasterServ
* The implementation of AsyncConnection.
*/
@InterfaceAudience.Private
-class AsyncConnectionImpl implements AsyncClusterConnection {
+class AsyncConnectionImpl implements AsyncConnection {
private static final Logger LOG =
LoggerFactory.getLogger(AsyncConnectionImpl.class);
@@ -85,7 +84,7 @@ class AsyncConnectionImpl implements AsyncClusterConnection {
private final int rpcTimeout;
- private final RpcClient rpcClient;
+ protected final RpcClient rpcClient;
final RpcControllerFactory rpcControllerFactory;
@@ -160,16 +159,10 @@ class AsyncConnectionImpl implements
AsyncClusterConnection {
}
// ditto
- @Override
- public NonceGenerator getNonceGenerator() {
+ NonceGenerator getNonceGenerator() {
return nonceGenerator;
}
- @Override
- public RpcClient getRpcClient() {
-return rpcClient;
- }
-
private ClientService.Interface createRegionServerStub(ServerName
serverName) throws IOException {
return ClientService.newStub(rpcClient.createRpcChannel(serverName, user,
rpcTimeout));
}
@@ -360,15 +353,4 @@ class AsyncConnectionImpl implements
AsyncClusterConnection {
return new HBaseHbck(MasterProtos.HbckService.newBlockingStub(
rpcClient.createBlockingRpcChannel(masterServer, user, rpcTimeout)),
rpcControllerFactory);
}
-
- public AsyncRegionServerAdmin getRegionServerAdmin(ServerName serverName) {
-return new AsyncRegionServerAdmin(serverName, this);
- }
-
- @Override
- public CompletableFuture flush(byte[] regionName,
- boolean writeFlushWALMarker) {
-RawAsyncHBaseAdmin admin = (RawAsyncHBaseAdmin) getAdmin();
-return admin.flushRegionInternal(regionName, writeFlushWALMarker);
- }
}
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
similarity index 72%
rename from
hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
rename to
hbase-server/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
index f1f64ca..0ad77ba 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
@@ -17,9 +17,13 @@
*/
package org.apache.hadoop.hbase.client;
[hbase] branch HBASE-20112 deleted (was 3a12912)
This is an automated email from the ASF dual-hosted git repository. psomogyi pushed a change to branch HBASE-20112 in repository https://gitbox.apache.org/repos/asf/hbase.git. was 3a12912 HBASE-20112 register nightly junit over hadoop3 results with jenkins. This change permanently discards the following revisions: discard 3a12912 HBASE-20112 register nightly junit over hadoop3 results with jenkins.
[hbase] branch HBASE-20364 deleted (was da8ede8)
This is an automated email from the ASF dual-hosted git repository. psomogyi pushed a change to branch HBASE-20364 in repository https://gitbox.apache.org/repos/asf/hbase.git. was da8ede8 WIP fail stages that do an scm checkout. This change permanently discards the following revisions: discard da8ede8 WIP fail stages that do an scm checkout. discard 7d04d37 HBASE-20364 ensure jira comment from nightly reflects results of run comment comes from.
[hbase] branch master updated: HBASE-21295 Update compatibility matrices
This is an automated email from the ASF dual-hosted git repository. psomogyi pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/master by this push: new 2b5b190 HBASE-21295 Update compatibility matrices 2b5b190 is described below commit 2b5b190882daeb0f9f0cf5361df3525db7119bea Author: Peter Somogyi AuthorDate: Wed Jan 9 13:59:30 2019 +0100 HBASE-21295 Update compatibility matrices Signed-off-by: Sean Busbey --- src/main/asciidoc/_chapters/configuration.adoc | 75 ++ 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/src/main/asciidoc/_chapters/configuration.adoc b/src/main/asciidoc/_chapters/configuration.adoc index b969c8b..545cefa 100644 --- a/src/main/asciidoc/_chapters/configuration.adoc +++ b/src/main/asciidoc/_chapters/configuration.adoc @@ -93,7 +93,10 @@ This section lists required services and some required system configuration. [[java]] .Java -The following table summarizes the recommendation of the HBase community wrt deploying on various Java versions. An entry of "yes" is meant to indicate a base level of testing and willingness to help diagnose and address issues you might run into. Similarly, an entry of "no" or "Not Supported" generally means that should you run into an issue the community is likely to ask you to change the Java environment before proceeding to help. In some cases, specific guidance on limitations (e.g. [...] +The following table summarizes the recommendation of the HBase community wrt deploying on various Java versions. +A icon:check-circle[role="green"] symbol is meant to indicate a base level of testing and willingness to help diagnose and address issues you might run into. +Similarly, an entry of icon:exclamation-circle[role="yellow"] or icon:times-circle[role="red"] generally means that should you run into an issue the community is likely to ask you to change the Java environment before proceeding to help. +In some cases, specific guidance on limitations (e.g. whether compiling / unit tests work, specific operational issues, etc) will also be noted. .Long Term Support JDKs are recommended [TIP] @@ -102,32 +105,34 @@ HBase recommends downstream users rely on JDK releases that are marked as Long T .Java support by release line -[cols="1,1,1,1,1", options="header"] +[cols="6*^.^", options="header"] |=== |HBase Version |JDK 7 |JDK 8 -|JDK 9 -|JDK 10 +|JDK 9 (Non-LTS) +|JDK 10 (Non-LTS) +|JDK 11 -|2.0 +|2.0+ |icon:times-circle[role="red"] |icon:check-circle[role="green"] -|icon:exclamation-circle[role="yellow"] Non-LTS, see link:https://issues.apache.org/jira/browse/HBASE-20264[HBASE-20264] -|icon:exclamation-circle[role="yellow"] Non-LTS, see link:https://issues.apache.org/jira/browse/HBASE-20264[HBASE-20264] - -|1.3 -|icon:check-circle[role="green"] -|icon:check-circle[role="green"] -|icon:exclamation-circle[role="yellow"] Non-LTS, see link:https://issues.apache.org/jira/browse/HBASE-20264[HBASE-20264] -|icon:exclamation-circle[role="yellow"] Non-LTS, see link:https://issues.apache.org/jira/browse/HBASE-20264[HBASE-20264] - - -|1.2 +v|icon:exclamation-circle[role="yellow"] +link:https://issues.apache.org/jira/browse/HBASE-20264[HBASE-20264] +v|icon:exclamation-circle[role="yellow"] +link:https://issues.apache.org/jira/browse/HBASE-20264[HBASE-20264] +v|icon:exclamation-circle[role="yellow"] +link:https://issues.apache.org/jira/browse/HBASE-21110[HBASE-21110] + +|1.2+ |icon:check-circle[role="green"] |icon:check-circle[role="green"] -|icon:exclamation-circle[role="yellow"] Non-LTS, see link:https://issues.apache.org/jira/browse/HBASE-20264[HBASE-20264] -|icon:exclamation-circle[role="yellow"] Non-LTS, see link:https://issues.apache.org/jira/browse/HBASE-20264[HBASE-20264] +v|icon:exclamation-circle[role="yellow"] +link:https://issues.apache.org/jira/browse/HBASE-20264[HBASE-20264] +v|icon:exclamation-circle[role="yellow"] +link:https://issues.apache.org/jira/browse/HBASE-20264[HBASE-20264] +v|icon:exclamation-circle[role="yellow"] +link:https://issues.apache.org/jira/browse/HBASE-21110[HBASE-21110] |=== @@ -217,24 +222,24 @@ Use the following legend to interpret this table: * icon:times-circle[role="red"] = Known to not be fully-functional * icon:exclamation-circle[role="yellow"] = Not tested, may/may-not function -[cols="1,1,1,1,1,1", options="header"] +[cols="1,4*^.^", options="header"] |=== -| | HBase-1.2.x | HBase-1.3.x | HBase-1.5.x | HBase-2.0.x | HBase-2.1.x -|Hadoop-2.4.x | icon:check-circle[role="green"] | icon:check-circle[role="green"] | icon:times-circle[role="red"] | icon:times-circle[role="red"] | icon:times-circle[role="red"] -|Hadoop-2.5.x | icon:check-circle[role="green"] | icon:check-circle[role="green"] | icon:times-circle[role="red"] | icon:times-circle[role="red"] | icon:times-circle[role="red"] -|Hadoop-2.6.0 | icon:times-circle[role="re
[hbase] branch master updated: HBASE-21704 The implementation of DistributedHBaseCluster.getServerHoldingRegion is incorrect
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 3d2580c HBASE-21704 The implementation of
DistributedHBaseCluster.getServerHoldingRegion is incorrect
3d2580c is described below
commit 3d2580cd6dbe32497543207ae616a9559de94739
Author: Duo Zhang
AuthorDate: Fri Jan 11 17:45:12 2019 +0800
HBASE-21704 The implementation of
DistributedHBaseCluster.getServerHoldingRegion is incorrect
---
.../hadoop/hbase/DistributedHBaseCluster.java | 47 +-
.../java/org/apache/hadoop/hbase/HBaseCluster.java | 5 ++-
2 files changed, 21 insertions(+), 31 deletions(-)
diff --git
a/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
b/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
index 5ec9e25..30a3db9 100644
---
a/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
+++
b/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
@@ -25,20 +25,19 @@ import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.TreeSet;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterManager.ServiceType;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
-import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ServerInfo;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MasterService;
@@ -312,20 +311,16 @@ public class DistributedHBaseCluster extends HBaseCluster
{
@Override
public ServerName getServerHoldingRegion(TableName tn, byte[] regionName)
throws IOException {
+byte[] startKey = RegionInfo.getStartKey(regionName);
HRegionLocation regionLoc = null;
try (RegionLocator locator = connection.getRegionLocator(tn)) {
- regionLoc = locator.getRegionLocation(regionName, true);
+ regionLoc = locator.getRegionLocation(startKey, true);
}
if (regionLoc == null) {
- LOG.warn("Cannot find region server holding region " +
Bytes.toString(regionName) +
-", start key [" + Bytes.toString(HRegionInfo.getStartKey(regionName))
+ "]");
+ LOG.warn("Cannot find region server holding region " +
Bytes.toStringBinary(regionName));
return null;
}
-
-AdminProtos.AdminService.BlockingInterface client =
-
((ClusterConnection)this.connection).getAdmin(regionLoc.getServerName());
-ServerInfo info = ProtobufUtil.getServerInfo(null, client);
-return ProtobufUtil.toServerName(info.getServerName());
+return regionLoc.getServerName();
}
@Override
@@ -366,17 +361,15 @@ public class DistributedHBaseCluster extends HBaseCluster
{
//check whether current master has changed
final ServerName initMaster = initial.getMasterName();
if (!ServerName.isSameAddress(initMaster, current.getMasterName())) {
- LOG.info("Restoring cluster - Initial active master : "
- + initMaster.getHostAndPort()
- + " has changed to : "
- + current.getMasterName().getHostAndPort());
+ LOG.info("Restoring cluster - Initial active master : " +
initMaster.getAddress() +
+" has changed to : " + current.getMasterName().getAddress());
// If initial master is stopped, start it, before restoring the state.
// It will come up as a backup master, if there is already an active
master.
try {
if (!clusterManager.isRunning(ServiceType.HBASE_MASTER,
initMaster.getHostname(), initMaster.getPort())) {
LOG.info("Restoring cluster - starting initial active master at:"
- + initMaster.getHostAndPort());
+ + initMaster.getAddress());
startMaster(initMaster.getHostname(), initMaster.getPort());
}
@@ -407,7 +400,7 @@ public class DistributedHBaseCluster extends HBaseCluster {
backup.getHostname(),
backup.getPort())) {
LOG.info("Restoring cluster - starting initial backup master: "
-+ backup.getHostAndPort());
++ backup.getAddress());
startMaster(backup.getHostname(), backup.getPort());
}
[hbase] branch branch-2 updated: HBASE-21704 The implementation of DistributedHBaseCluster.getServerHoldingRegion is incorrect
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 9ec84c2 HBASE-21704 The implementation of
DistributedHBaseCluster.getServerHoldingRegion is incorrect
9ec84c2 is described below
commit 9ec84c235f135d40dfdcfb0a629648772b88fb8e
Author: Duo Zhang
AuthorDate: Fri Jan 11 17:45:12 2019 +0800
HBASE-21704 The implementation of
DistributedHBaseCluster.getServerHoldingRegion is incorrect
---
.../hadoop/hbase/DistributedHBaseCluster.java | 47 +-
.../java/org/apache/hadoop/hbase/HBaseCluster.java | 5 ++-
2 files changed, 21 insertions(+), 31 deletions(-)
diff --git
a/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
b/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
index 5ec9e25..30a3db9 100644
---
a/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
+++
b/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
@@ -25,20 +25,19 @@ import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.TreeSet;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterManager.ServiceType;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
-import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ServerInfo;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MasterService;
@@ -312,20 +311,16 @@ public class DistributedHBaseCluster extends HBaseCluster
{
@Override
public ServerName getServerHoldingRegion(TableName tn, byte[] regionName)
throws IOException {
+byte[] startKey = RegionInfo.getStartKey(regionName);
HRegionLocation regionLoc = null;
try (RegionLocator locator = connection.getRegionLocator(tn)) {
- regionLoc = locator.getRegionLocation(regionName, true);
+ regionLoc = locator.getRegionLocation(startKey, true);
}
if (regionLoc == null) {
- LOG.warn("Cannot find region server holding region " +
Bytes.toString(regionName) +
-", start key [" + Bytes.toString(HRegionInfo.getStartKey(regionName))
+ "]");
+ LOG.warn("Cannot find region server holding region " +
Bytes.toStringBinary(regionName));
return null;
}
-
-AdminProtos.AdminService.BlockingInterface client =
-
((ClusterConnection)this.connection).getAdmin(regionLoc.getServerName());
-ServerInfo info = ProtobufUtil.getServerInfo(null, client);
-return ProtobufUtil.toServerName(info.getServerName());
+return regionLoc.getServerName();
}
@Override
@@ -366,17 +361,15 @@ public class DistributedHBaseCluster extends HBaseCluster
{
//check whether current master has changed
final ServerName initMaster = initial.getMasterName();
if (!ServerName.isSameAddress(initMaster, current.getMasterName())) {
- LOG.info("Restoring cluster - Initial active master : "
- + initMaster.getHostAndPort()
- + " has changed to : "
- + current.getMasterName().getHostAndPort());
+ LOG.info("Restoring cluster - Initial active master : " +
initMaster.getAddress() +
+" has changed to : " + current.getMasterName().getAddress());
// If initial master is stopped, start it, before restoring the state.
// It will come up as a backup master, if there is already an active
master.
try {
if (!clusterManager.isRunning(ServiceType.HBASE_MASTER,
initMaster.getHostname(), initMaster.getPort())) {
LOG.info("Restoring cluster - starting initial active master at:"
- + initMaster.getHostAndPort());
+ + initMaster.getAddress());
startMaster(initMaster.getHostname(), initMaster.getPort());
}
@@ -407,7 +400,7 @@ public class DistributedHBaseCluster extends HBaseCluster {
backup.getHostname(),
backup.getPort())) {
LOG.info("Restoring cluster - starting initial backup master: "
-+ backup.getHostAndPort());
++ backup.getAddress());
startMaster(backup.getHostname(), backup.getPort());
[hbase] branch branch-2.1 updated: HBASE-21704 The implementation of DistributedHBaseCluster.getServerHoldingRegion is incorrect
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new d7db78b HBASE-21704 The implementation of
DistributedHBaseCluster.getServerHoldingRegion is incorrect
d7db78b is described below
commit d7db78b74ca586536cf73894ea6779716132e26f
Author: Duo Zhang
AuthorDate: Fri Jan 11 17:45:12 2019 +0800
HBASE-21704 The implementation of
DistributedHBaseCluster.getServerHoldingRegion is incorrect
---
.../hadoop/hbase/DistributedHBaseCluster.java | 45 --
.../java/org/apache/hadoop/hbase/HBaseCluster.java | 5 ++-
2 files changed, 20 insertions(+), 30 deletions(-)
diff --git
a/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
b/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
index 943f2a6..1e37d24 100644
---
a/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
+++
b/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
@@ -31,14 +31,13 @@ import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
-import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ServerInfo;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MasterService;
@@ -281,20 +280,16 @@ public class DistributedHBaseCluster extends HBaseCluster
{
@Override
public ServerName getServerHoldingRegion(TableName tn, byte[] regionName)
throws IOException {
+byte[] startKey = RegionInfo.getStartKey(regionName);
HRegionLocation regionLoc = null;
try (RegionLocator locator = connection.getRegionLocator(tn)) {
- regionLoc = locator.getRegionLocation(regionName, true);
+ regionLoc = locator.getRegionLocation(startKey, true);
}
if (regionLoc == null) {
- LOG.warn("Cannot find region server holding region " +
Bytes.toString(regionName) +
-", start key [" + Bytes.toString(HRegionInfo.getStartKey(regionName))
+ "]");
+ LOG.warn("Cannot find region server holding region " +
Bytes.toStringBinary(regionName));
return null;
}
-
-AdminProtos.AdminService.BlockingInterface client =
-
((ClusterConnection)this.connection).getAdmin(regionLoc.getServerName());
-ServerInfo info = ProtobufUtil.getServerInfo(null, client);
-return ProtobufUtil.toServerName(info.getServerName());
+return regionLoc.getServerName();
}
@Override
@@ -335,17 +330,15 @@ public class DistributedHBaseCluster extends HBaseCluster
{
//check whether current master has changed
final ServerName initMaster = initial.getMasterName();
if (!ServerName.isSameAddress(initMaster, current.getMasterName())) {
- LOG.info("Restoring cluster - Initial active master : "
- + initMaster.getHostAndPort()
- + " has changed to : "
- + current.getMasterName().getHostAndPort());
+ LOG.info("Restoring cluster - Initial active master : " +
initMaster.getAddress() +
+" has changed to : " + current.getMasterName().getAddress());
// If initial master is stopped, start it, before restoring the state.
// It will come up as a backup master, if there is already an active
master.
try {
if (!clusterManager.isRunning(ServiceType.HBASE_MASTER,
initMaster.getHostname(), initMaster.getPort())) {
LOG.info("Restoring cluster - starting initial active master at:"
- + initMaster.getHostAndPort());
+ + initMaster.getAddress());
startMaster(initMaster.getHostname(), initMaster.getPort());
}
@@ -376,7 +369,7 @@ public class DistributedHBaseCluster extends HBaseCluster {
backup.getHostname(),
backup.getPort())) {
LOG.info("Restoring cluster - starting initial backup master: "
-+ backup.getHostAndPort());
++ backup.getAddress());
startMaster(backup.getHostname(), backup.getPort());
}
} catch (IOException ex) {
@@ -400,7 +393,7 @@ public class DistributedHBaseCluster extends HBaseCluster {
for (ServerName sn:toStart) {
try {
if(!clusterManager.is
[hbase] branch branch-2.0 updated: HBASE-21704 The implementation of DistributedHBaseCluster.getServerHoldingRegion is incorrect
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new a8905cf HBASE-21704 The implementation of
DistributedHBaseCluster.getServerHoldingRegion is incorrect
a8905cf is described below
commit a8905cfffae7ef7707aad0351b35b9dbbcdb9e44
Author: Duo Zhang
AuthorDate: Fri Jan 11 17:45:12 2019 +0800
HBASE-21704 The implementation of
DistributedHBaseCluster.getServerHoldingRegion is incorrect
---
.../hadoop/hbase/DistributedHBaseCluster.java | 45 --
.../java/org/apache/hadoop/hbase/HBaseCluster.java | 5 ++-
2 files changed, 20 insertions(+), 30 deletions(-)
diff --git
a/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
b/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
index 943f2a6..1e37d24 100644
---
a/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
+++
b/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
@@ -31,14 +31,13 @@ import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
-import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ServerInfo;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MasterService;
@@ -281,20 +280,16 @@ public class DistributedHBaseCluster extends HBaseCluster
{
@Override
public ServerName getServerHoldingRegion(TableName tn, byte[] regionName)
throws IOException {
+byte[] startKey = RegionInfo.getStartKey(regionName);
HRegionLocation regionLoc = null;
try (RegionLocator locator = connection.getRegionLocator(tn)) {
- regionLoc = locator.getRegionLocation(regionName, true);
+ regionLoc = locator.getRegionLocation(startKey, true);
}
if (regionLoc == null) {
- LOG.warn("Cannot find region server holding region " +
Bytes.toString(regionName) +
-", start key [" + Bytes.toString(HRegionInfo.getStartKey(regionName))
+ "]");
+ LOG.warn("Cannot find region server holding region " +
Bytes.toStringBinary(regionName));
return null;
}
-
-AdminProtos.AdminService.BlockingInterface client =
-
((ClusterConnection)this.connection).getAdmin(regionLoc.getServerName());
-ServerInfo info = ProtobufUtil.getServerInfo(null, client);
-return ProtobufUtil.toServerName(info.getServerName());
+return regionLoc.getServerName();
}
@Override
@@ -335,17 +330,15 @@ public class DistributedHBaseCluster extends HBaseCluster
{
//check whether current master has changed
final ServerName initMaster = initial.getMasterName();
if (!ServerName.isSameAddress(initMaster, current.getMasterName())) {
- LOG.info("Restoring cluster - Initial active master : "
- + initMaster.getHostAndPort()
- + " has changed to : "
- + current.getMasterName().getHostAndPort());
+ LOG.info("Restoring cluster - Initial active master : " +
initMaster.getAddress() +
+" has changed to : " + current.getMasterName().getAddress());
// If initial master is stopped, start it, before restoring the state.
// It will come up as a backup master, if there is already an active
master.
try {
if (!clusterManager.isRunning(ServiceType.HBASE_MASTER,
initMaster.getHostname(), initMaster.getPort())) {
LOG.info("Restoring cluster - starting initial active master at:"
- + initMaster.getHostAndPort());
+ + initMaster.getAddress());
startMaster(initMaster.getHostname(), initMaster.getPort());
}
@@ -376,7 +369,7 @@ public class DistributedHBaseCluster extends HBaseCluster {
backup.getHostname(),
backup.getPort())) {
LOG.info("Restoring cluster - starting initial backup master: "
-+ backup.getHostAndPort());
++ backup.getAddress());
startMaster(backup.getHostname(), backup.getPort());
}
} catch (IOException ex) {
@@ -400,7 +393,7 @@ public class DistributedHBaseCluster extends HBaseCluster {
for (ServerName sn:toStart) {
try {
if(!clusterManager.is
[hbase] 06/06: HBASE-21671 Rewrite RegionReplicaReplicationEndpoint to use AsyncClusterConnection
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-21512
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 199c994a5c2983d9a8c6185913dc673fdf26e9c3
Author: Duo Zhang
AuthorDate: Fri Jan 11 16:22:24 2019 +0800
HBASE-21671 Rewrite RegionReplicaReplicationEndpoint to use
AsyncClusterConnection
---
.../hadoop/hbase/client/AsyncConnectionImpl.java | 24 +-
.../hbase/client/AsyncClusterConnection.java | 17 +
.../hbase/client/AsyncClusterConnectionImpl.java | 80 +++
.../AsyncRegionReplicaReplayRetryingCaller.java| 146
.../hbase/client/AsyncRegionServerAdmin.java | 5 +-
.../hbase/client/ClusterConnectionFactory.java | 2 +-
.../hbase/protobuf/ReplicationProtbufUtil.java | 31 +-
.../handler/RegionReplicaFlushHandler.java | 3 +-
.../hbase/replication/ReplicationEndpoint.java | 35 +-
.../RegionReplicaReplicationEndpoint.java | 783 +++--
.../regionserver/ReplicationSource.java| 2 +-
.../hbase/client/TestAsyncTableNoncedRetry.java| 2 +-
.../TestRegionReplicaReplicationEndpoint.java | 56 +-
...stRegionReplicaReplicationEndpointNoMaster.java | 99 ++-
14 files changed, 627 insertions(+), 658 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
index ce6bfac..f6a2149 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
@@ -55,7 +55,6 @@ import
org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer;
import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
-import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ClientService;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsMasterRunningResponse;
@@ -65,7 +64,7 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MasterServ
* The implementation of AsyncConnection.
*/
@InterfaceAudience.Private
-class AsyncConnectionImpl implements AsyncClusterConnection {
+class AsyncConnectionImpl implements AsyncConnection {
private static final Logger LOG =
LoggerFactory.getLogger(AsyncConnectionImpl.class);
@@ -85,7 +84,7 @@ class AsyncConnectionImpl implements AsyncClusterConnection {
private final int rpcTimeout;
- private final RpcClient rpcClient;
+ protected final RpcClient rpcClient;
final RpcControllerFactory rpcControllerFactory;
@@ -160,16 +159,10 @@ class AsyncConnectionImpl implements
AsyncClusterConnection {
}
// ditto
- @Override
- public NonceGenerator getNonceGenerator() {
+ NonceGenerator getNonceGenerator() {
return nonceGenerator;
}
- @Override
- public RpcClient getRpcClient() {
-return rpcClient;
- }
-
private ClientService.Interface createRegionServerStub(ServerName
serverName) throws IOException {
return ClientService.newStub(rpcClient.createRpcChannel(serverName, user,
rpcTimeout));
}
@@ -360,15 +353,4 @@ class AsyncConnectionImpl implements
AsyncClusterConnection {
return new HBaseHbck(MasterProtos.HbckService.newBlockingStub(
rpcClient.createBlockingRpcChannel(masterServer, user, rpcTimeout)),
rpcControllerFactory);
}
-
- public AsyncRegionServerAdmin getRegionServerAdmin(ServerName serverName) {
-return new AsyncRegionServerAdmin(serverName, this);
- }
-
- @Override
- public CompletableFuture flush(byte[] regionName,
- boolean writeFlushWALMarker) {
-RawAsyncHBaseAdmin admin = (RawAsyncHBaseAdmin) getAdmin();
-return admin.flushRegionInternal(regionName, writeFlushWALMarker);
- }
}
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
similarity index 72%
rename from
hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
rename to
hbase-server/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
index f1f64ca..0ad77ba 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
@@ -17,9 +17,13 @@
*/
package org.apache.hadoop.hbase.client;
+import java.util.List;
import java.util.concurrent.CompletableFuture;
+import org.apache.hadoop.hbase.RegionLocations;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableN
[hbase] 03/06: HBASE-21526 Use AsyncClusterConnection in ServerManager for getRsAdmin
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-21512
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit e0c580dcc86f220d364589cf9ad9bd034d11193b
Author: zhangduo
AuthorDate: Thu Dec 6 21:25:34 2018 +0800
HBASE-21526 Use AsyncClusterConnection in ServerManager for getRsAdmin
---
.../hbase/client/AsyncClusterConnection.java | 6 +
.../hadoop/hbase/client/AsyncConnectionImpl.java | 4 +
.../hbase/client/AsyncRegionServerAdmin.java | 210 +
.../org/apache/hadoop/hbase/util/FutureUtils.java | 2 +-
.../org/apache/hadoop/hbase/master/HMaster.java| 15 +-
.../apache/hadoop/hbase/master/ServerManager.java | 67 ---
.../master/procedure/RSProcedureDispatcher.java| 44 +++--
7 files changed, 262 insertions(+), 86 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
index c7dea25..1327fd7 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.client;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.ipc.RpcClient;
import org.apache.yetus.audience.InterfaceAudience;
@@ -27,6 +28,11 @@ import org.apache.yetus.audience.InterfaceAudience;
public interface AsyncClusterConnection extends AsyncConnection {
/**
+ * Get the admin service for the given region server.
+ */
+ AsyncRegionServerAdmin getRegionServerAdmin(ServerName serverName);
+
+ /**
* Get the nonce generator for this connection.
*/
NonceGenerator getNonceGenerator();
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
index 50e27c4..9bead83 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
@@ -359,4 +359,8 @@ class AsyncConnectionImpl implements AsyncClusterConnection
{
return new HBaseHbck(MasterProtos.HbckService.newBlockingStub(
rpcClient.createBlockingRpcChannel(masterServer, user, rpcTimeout)),
rpcControllerFactory);
}
+
+ public AsyncRegionServerAdmin getRegionServerAdmin(ServerName serverName) {
+return new AsyncRegionServerAdmin(serverName, this);
+ }
}
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java
new file mode 100644
index 000..9accd89
--- /dev/null
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java
@@ -0,0 +1,210 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import java.io.IOException;
+import java.util.concurrent.CompletableFuture;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.ipc.HBaseRpcController;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback;
+import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
+
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesRequest;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesResponse;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheRequest;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheResponse;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionResponse;
+import
org.apache.hadoop.hbase.shaded.protobuf.ge
[hbase] 05/06: HBASE-21538 Rewrite RegionReplicaFlushHandler to use AsyncClusterConnection
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-21512
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit a2453f4e3cace29036c6f1a936926e5def2b11ce
Author: Duo Zhang
AuthorDate: Wed Dec 12 09:33:33 2018 +0800
HBASE-21538 Rewrite RegionReplicaFlushHandler to use AsyncClusterConnection
---
.../hbase/client/AsyncClusterConnection.java | 8 ++
.../hadoop/hbase/client/AsyncConnectionImpl.java | 8 ++
.../hbase/client/ClusterConnectionFactory.java | 16 +--
.../hadoop/hbase/client/RawAsyncHBaseAdmin.java| 36 ---
.../org/apache/hadoop/hbase/util/FutureUtils.java | 22 +
.../master/procedure/RSProcedureDispatcher.java| 34 +--
.../hbase/protobuf/ReplicationProtbufUtil.java | 15 +--
.../hadoop/hbase/regionserver/HRegionServer.java | 3 +-
.../handler/RegionReplicaFlushHandler.java | 110 -
9 files changed, 132 insertions(+), 120 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
index 1327fd7..f1f64ca 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
@@ -17,10 +17,13 @@
*/
package org.apache.hadoop.hbase.client;
+import java.util.concurrent.CompletableFuture;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.ipc.RpcClient;
import org.apache.yetus.audience.InterfaceAudience;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse;
+
/**
* The asynchronous connection for internal usage.
*/
@@ -41,4 +44,9 @@ public interface AsyncClusterConnection extends
AsyncConnection {
* Get the rpc client we used to communicate with other servers.
*/
RpcClient getRpcClient();
+
+ /**
+ * Flush a region and get the response.
+ */
+ CompletableFuture flush(byte[] regionName, boolean
writeFlushWALMarker);
}
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
index 9bead83..ce6bfac 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
@@ -55,6 +55,7 @@ import
org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer;
import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ClientService;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsMasterRunningResponse;
@@ -363,4 +364,11 @@ class AsyncConnectionImpl implements
AsyncClusterConnection {
public AsyncRegionServerAdmin getRegionServerAdmin(ServerName serverName) {
return new AsyncRegionServerAdmin(serverName, this);
}
+
+ @Override
+ public CompletableFuture flush(byte[] regionName,
+ boolean writeFlushWALMarker) {
+RawAsyncHBaseAdmin admin = (RawAsyncHBaseAdmin) getAdmin();
+return admin.flushRegionInternal(regionName, writeFlushWALMarker);
+ }
}
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterConnectionFactory.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterConnectionFactory.java
index 68c0630..79484db 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterConnectionFactory.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterConnectionFactory.java
@@ -18,15 +18,12 @@
package org.apache.hadoop.hbase.client;
import java.io.IOException;
-import java.io.InterruptedIOException;
import java.net.SocketAddress;
-import java.util.concurrent.ExecutionException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.util.FutureUtils;
import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hbase.thirdparty.com.google.common.base.Throwables;
-
/**
* The factory for creating {@link AsyncClusterConnection}.
*/
@@ -48,16 +45,7 @@ public final class ClusterConnectionFactory {
public static AsyncClusterConnection
createAsyncClusterConnection(Configuration conf,
SocketAddress localAddress, User user) throws IOException {
AsyncRegistry registry = AsyncRegistryFactory.getRegistry(conf);
-String clusterId;
-try {
- clusterId = registry.getClusterId().get();
-} catch (InterruptedException
[hbase] 01/06: HBASE-21515 Also initialize an AsyncClusterConnection in HRegionServer
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-21512
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 16c0c32b09e742a926645d5be3eaaec63c9c99e6
Author: zhangduo
AuthorDate: Fri Nov 30 08:23:47 2018 +0800
HBASE-21515 Also initialize an AsyncClusterConnection in HRegionServer
---
.../hbase/client/AsyncClusterConnection.java | 38 +
.../hadoop/hbase/client/AsyncConnectionImpl.java | 14 +++--
.../hbase/client/ClusterConnectionFactory.java | 63 ++
.../hadoop/hbase/client/ConnectionFactory.java | 5 +-
.../apache/hadoop/hbase/util/ReflectionUtils.java | 22
.../main/java/org/apache/hadoop/hbase/Server.java | 20 +++
.../org/apache/hadoop/hbase/master/HMaster.java| 3 ++
.../hadoop/hbase/regionserver/HRegionServer.java | 56 +--
.../regionserver/ReplicationSyncUp.java| 6 +++
.../hadoop/hbase/MockRegionServerServices.java | 5 ++
.../client/TestAsyncNonMetaRegionLocator.java | 2 +-
...stAsyncNonMetaRegionLocatorConcurrenyLimit.java | 2 +-
.../client/TestAsyncRegionLocatorTimeout.java | 2 +-
.../TestAsyncSingleRequestRpcRetryingCaller.java | 4 +-
.../hbase/client/TestAsyncTableNoncedRetry.java| 2 +-
.../hbase/master/MockNoopMasterServices.java | 6 +++
.../hadoop/hbase/master/MockRegionServer.java | 5 ++
.../hbase/master/TestActiveMasterManager.java | 6 +++
.../hbase/master/cleaner/TestHFileCleaner.java | 6 +++
.../hbase/master/cleaner/TestHFileLinkCleaner.java | 6 +++
.../hbase/master/cleaner/TestLogsCleaner.java | 6 +++
.../cleaner/TestReplicationHFileCleaner.java | 6 +++
.../hbase/regionserver/TestHeapMemoryManager.java | 6 +++
.../hbase/regionserver/TestSplitLogWorker.java | 6 +++
.../hadoop/hbase/regionserver/TestWALLockup.java | 6 +++
.../replication/TestReplicationTrackerZKImpl.java | 6 +++
.../regionserver/TestReplicationSourceManager.java | 6 +++
.../security/token/TestTokenAuthentication.java| 6 +++
.../org/apache/hadoop/hbase/util/MockServer.java | 6 +++
29 files changed, 290 insertions(+), 37 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
new file mode 100644
index 000..c7dea25
--- /dev/null
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import org.apache.hadoop.hbase.ipc.RpcClient;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * The asynchronous connection for internal usage.
+ */
+@InterfaceAudience.Private
+public interface AsyncClusterConnection extends AsyncConnection {
+
+ /**
+ * Get the nonce generator for this connection.
+ */
+ NonceGenerator getNonceGenerator();
+
+ /**
+ * Get the rpc client we used to communicate with other servers.
+ */
+ RpcClient getRpcClient();
+}
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
index 3cbd950..50e27c4 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
@@ -23,6 +23,7 @@ import static
org.apache.hadoop.hbase.client.NonceGenerator.CLIENT_NONCES_ENABLE
import static org.apache.hadoop.hbase.util.FutureUtils.addListener;
import java.io.IOException;
+import java.net.SocketAddress;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -63,7 +64,7 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MasterServ
* The implementation of AsyncConnection.
*/
@InterfaceAudience.Private
-class AsyncConnectionImpl implements AsyncConnection {
+class AsyncConnectionImpl implements AsyncClusterConnecti
[hbase] 02/06: HBASE-21516 Use AsyncConnection instead of Connection in SecureBulkLoadManager
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-21512
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit ec42d7ec75be580996fa5853987d5e85707f9d51
Author: zhangduo
AuthorDate: Sat Dec 1 21:15:48 2018 +0800
HBASE-21516 Use AsyncConnection instead of Connection in
SecureBulkLoadManager
---
.../apache/hadoop/hbase/protobuf/ProtobufUtil.java | 5 +-
.../hadoop/hbase/shaded/protobuf/ProtobufUtil.java | 7 ++-
.../hadoop/hbase/regionserver/HRegionServer.java | 2 +-
.../hbase/regionserver/SecureBulkLoadManager.java | 24 -
.../hadoop/hbase/security/token/TokenUtil.java | 57 +-
.../hadoop/hbase/security/token/TestTokenUtil.java | 42
6 files changed, 96 insertions(+), 41 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index a3d49b5..d9e620b 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -261,13 +261,12 @@ public final class ProtobufUtil {
* just {@link ServiceException}. Prefer this method to
* {@link #getRemoteException(ServiceException)} because trying to
* contain direct protobuf references.
- * @param e
*/
- public static IOException handleRemoteException(Exception e) {
+ public static IOException handleRemoteException(Throwable e) {
return makeIOExceptionOfException(e);
}
- private static IOException makeIOExceptionOfException(Exception e) {
+ private static IOException makeIOExceptionOfException(Throwable e) {
Throwable t = e;
if (e instanceof ServiceException ||
e instanceof
org.apache.hbase.thirdparty.com.google.protobuf.ServiceException) {
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
index fea81f1..de2fb7d 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
@@ -40,7 +40,6 @@ import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ByteBufferExtendedCell;
@@ -123,6 +122,7 @@ import
org.apache.hbase.thirdparty.com.google.protobuf.Service;
import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
import org.apache.hbase.thirdparty.com.google.protobuf.TextFormat;
import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
+
import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
@@ -343,13 +343,12 @@ public final class ProtobufUtil {
* just {@link ServiceException}. Prefer this method to
* {@link #getRemoteException(ServiceException)} because trying to
* contain direct protobuf references.
- * @param e
*/
- public static IOException handleRemoteException(Exception e) {
+ public static IOException handleRemoteException(Throwable e) {
return makeIOExceptionOfException(e);
}
- private static IOException makeIOExceptionOfException(Exception e) {
+ private static IOException makeIOExceptionOfException(Throwable e) {
Throwable t = e;
if (e instanceof ServiceException) {
t = e.getCause();
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index b4b1d3e..bbc9d3c 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1937,7 +1937,7 @@ public class HRegionServer extends HasThread implements
if (!isStopped() && !isAborted()) {
initializeThreads();
}
-this.secureBulkLoadManager = new SecureBulkLoadManager(this.conf,
clusterConnection);
+this.secureBulkLoadManager = new SecureBulkLoadManager(this.conf,
asyncClusterConnection);
this.secureBulkLoadManager.start();
// Health checker thread.
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java
index 566a6b6..add6519 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBul
[hbase] 04/06: HBASE-21579 Use AsyncClusterConnection for HBaseInterClusterReplicationEndpoint
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-21512
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 132fe1c1833584bfb09e4a1e18b13e5e7b5a9198
Author: zhangduo
AuthorDate: Tue Jan 1 21:27:14 2019 +0800
HBASE-21579 Use AsyncClusterConnection for
HBaseInterClusterReplicationEndpoint
---
.../hbase/client/AsyncRegionServerAdmin.java | 14 +---
.../hbase/protobuf/ReplicationProtbufUtil.java | 35 ++-
.../HBaseInterClusterReplicationEndpoint.java | 31 +
.../regionserver/ReplicationSinkManager.java | 40 --
.../hbase/replication/SyncReplicationTestBase.java | 12 +++
.../regionserver/TestReplicationSinkManager.java | 21 +---
6 files changed, 74 insertions(+), 79 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java
index 9accd89..b9141a9 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.client;
import java.io.IOException;
import java.util.concurrent.CompletableFuture;
+import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.ipc.HBaseRpcController;
import org.apache.yetus.audience.InterfaceAudience;
@@ -94,9 +95,9 @@ public class AsyncRegionServerAdmin {
void call(AdminService.Interface stub, HBaseRpcController controller,
RpcCallback done);
}
- private CompletableFuture call(RpcCall rpcCall) {
+ private CompletableFuture call(RpcCall rpcCall,
CellScanner cellScanner) {
CompletableFuture future = new CompletableFuture<>();
-HBaseRpcController controller = conn.rpcControllerFactory.newController();
+HBaseRpcController controller =
conn.rpcControllerFactory.newController(cellScanner);
try {
rpcCall.call(conn.getAdminStub(server), controller, new
RpcCallback() {
@@ -115,6 +116,10 @@ public class AsyncRegionServerAdmin {
return future;
}
+ private CompletableFuture call(RpcCall rpcCall) {
+return call(rpcCall, null);
+ }
+
public CompletableFuture
getRegionInfo(GetRegionInfoRequest request) {
return call((stub, controller, done) -> stub.getRegionInfo(controller,
request, done));
}
@@ -154,8 +159,9 @@ public class AsyncRegionServerAdmin {
}
public CompletableFuture replicateWALEntry(
- ReplicateWALEntryRequest request) {
-return call((stub, controller, done) -> stub.replicateWALEntry(controller,
request, done));
+ ReplicateWALEntryRequest request, CellScanner cellScanner) {
+return call((stub, controller, done) -> stub.replicateWALEntry(controller,
request, done),
+ cellScanner);
}
public CompletableFuture
replay(ReplicateWALEntryRequest request) {
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java
index c1b3911..74fad26 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java
@@ -20,51 +20,54 @@ package org.apache.hadoop.hbase.protobuf;
import java.io.IOException;
+import java.io.InterruptedIOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import java.util.concurrent.ExecutionException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.PrivateCellUtil;
+import org.apache.hadoop.hbase.client.AsyncRegionServerAdmin;
import org.apache.hadoop.hbase.io.SizedCellScanner;
-import org.apache.hadoop.hbase.ipc.HBaseRpcController;
-import org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl;
import org.apache.hadoop.hbase.regionserver.wal.WALCellCodec;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.wal.WAL.Entry;
import org.apache.hadoop.hbase.wal.WALEdit;
import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.hbase.thirdparty.com.google.common.base.Throwables;
import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
-import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;
@InterfaceAudience.Private
public class ReplicationProtbufUtil {
+
/**
- * A helper to replicate a list of WAL entries using admin protocol.
- * @param admin Admin service
+ * A helper to r
[hbase] branch HBASE-21512 updated (d6fbe51 -> 199c994)
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a change to branch HBASE-21512 in repository https://gitbox.apache.org/repos/asf/hbase.git. discard d6fbe51 HBASE-21671 Rewrite RegionReplicaReplicationEndpoint to use AsyncClusterConnection discard 19c9908 HBASE-21538 Rewrite RegionReplicaFlushHandler to use AsyncClusterConnection discard 314094b HBASE-21579 Use AsyncClusterConnection for HBaseInterClusterReplicationEndpoint discard b542ca8 HBASE-21526 Use AsyncClusterConnection in ServerManager for getRsAdmin discard ebf6f9c HBASE-21516 Use AsyncConnection instead of Connection in SecureBulkLoadManager discard f9914c5 HBASE-21515 Also initialize an AsyncClusterConnection in HRegionServer add 2b5b190 HBASE-21295 Update compatibility matrices add 3d2580c HBASE-21704 The implementation of DistributedHBaseCluster.getServerHoldingRegion is incorrect new 16c0c32 HBASE-21515 Also initialize an AsyncClusterConnection in HRegionServer new ec42d7e HBASE-21516 Use AsyncConnection instead of Connection in SecureBulkLoadManager new e0c580d HBASE-21526 Use AsyncClusterConnection in ServerManager for getRsAdmin new 132fe1c HBASE-21579 Use AsyncClusterConnection for HBaseInterClusterReplicationEndpoint new a2453f4 HBASE-21538 Rewrite RegionReplicaFlushHandler to use AsyncClusterConnection new 199c994 HBASE-21671 Rewrite RegionReplicaReplicationEndpoint to use AsyncClusterConnection This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (d6fbe51) \ N -- N -- N refs/heads/HBASE-21512 (199c994) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. The 6 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../hadoop/hbase/DistributedHBaseCluster.java | 47 ++ .../java/org/apache/hadoop/hbase/HBaseCluster.java | 5 +- src/main/asciidoc/_chapters/configuration.adoc | 75 -- 3 files changed, 61 insertions(+), 66 deletions(-)
[hbase-site] branch asf-site updated: INFRA-10751 Empty commit
This is an automated email from the ASF dual-hosted git repository. git-site-role pushed a commit to branch asf-site in repository https://gitbox.apache.org/repos/asf/hbase-site.git The following commit(s) were added to refs/heads/asf-site by this push: new 8c35057 INFRA-10751 Empty commit 8c35057 is described below commit 8c3505755ac66304539245259b4f7cc484534e29 Author: jenkins AuthorDate: Fri Jan 11 14:48:17 2019 + INFRA-10751 Empty commit
[hbase] 02/04: HBASE-21208 Bytes#toShort doesn't work without unsafe
This is an automated email from the ASF dual-hosted git repository.
apurtell pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 2f541f5710784877787d47f2b0036561e5aff160
Author: Chia-Ping Tsai
AuthorDate: Tue Sep 25 10:18:47 2018 +0800
HBASE-21208 Bytes#toShort doesn't work without unsafe
Signed-off-by: Ted Yu
Signed-off-by: anoopsamjohn
Signed-off-by: Reid Chan
Conflicts:
hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java
Amending-Author: Andrew Purtell
---
.../java/org/apache/hadoop/hbase/util/Bytes.java | 13 ++-
.../org/apache/hadoop/hbase/util/TestBytes.java| 45 ++
2 files changed, 56 insertions(+), 2 deletions(-)
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
index 8e55b63..765f51b 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
@@ -129,7 +129,8 @@ public class Bytes implements Comparable {
// SizeOf which uses java.lang.instrument says 24 bytes. (3 longs?)
public static final int ESTIMATED_HEAP_TAX = 16;
- private static final boolean UNSAFE_UNALIGNED =
UnsafeAvailChecker.unaligned();
+ @VisibleForTesting
+ static final boolean UNSAFE_UNALIGNED = UnsafeAvailChecker.unaligned();
/**
* Returns length of the byte array, returning 0 if the array is null.
@@ -1082,7 +1083,15 @@ public class Bytes implements Comparable {
if (length != SIZEOF_SHORT || offset + length > bytes.length) {
throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT);
}
-return ConverterHolder.BEST_CONVERTER.toShort(bytes, offset, length);
+if (UNSAFE_UNALIGNED) {
+ return ConverterHolder.BEST_CONVERTER.toShort(bytes, offset, length);
+} else {
+ short n = 0;
+ n = (short) (n ^ (bytes[offset] & 0xFF));
+ n = (short) (n << 8);
+ n = (short) (n ^ (bytes[offset + 1] & 0xFF));
+ return n;
+}
}
/**
diff --git
a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java
b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java
index 7771f87..2e19ddd 100644
--- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java
+++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java
@@ -22,6 +22,8 @@ import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -39,6 +41,49 @@ import org.junit.experimental.categories.Category;
@Category(SmallTests.class)
public class TestBytes extends TestCase {
+
+ private static void setUnsafe(boolean value) throws Exception {
+Field field = Bytes.class.getDeclaredField("UNSAFE_UNALIGNED");
+field.setAccessible(true);
+Field modifiersField = Field.class.getDeclaredField("modifiers");
+modifiersField.setAccessible(true);
+int oldModifiers = field.getModifiers();
+modifiersField.setInt(field, oldModifiers & ~Modifier.FINAL);
+try {
+ field.set(null, value);
+} finally {
+ modifiersField.setInt(field, oldModifiers);
+}
+assertEquals(Bytes.UNSAFE_UNALIGNED, value);
+ }
+
+ public void testShort() throws Exception {
+testShort(false);
+ }
+
+ public void testShortUnsafe() throws Exception {
+testShort(true);
+ }
+
+ private static void testShort(boolean unsafe) throws Exception {
+setUnsafe(unsafe);
+try {
+ for (short n : Arrays.asList(
+ Short.MIN_VALUE,
+ (short) -100,
+ (short) -1,
+ (short) 0,
+ (short) 1,
+ (short) 300,
+ Short.MAX_VALUE)) {
+byte[] bytes = Bytes.toBytes(n);
+assertEquals(Bytes.toShort(bytes, 0, bytes.length), n);
+ }
+} finally {
+ setUnsafe(UnsafeAvailChecker.unaligned());
+}
+ }
+
public void testNullHashCode() {
byte [] b = null;
Exception ee = null;
[hbase] branch branch-1 updated (582c649 -> f3f3798)
This is an automated email from the ASF dual-hosted git repository. apurtell pushed a change to branch branch-1 in repository https://gitbox.apache.org/repos/asf/hbase.git. from 582c649 HBASE-21374 Backport HBASE-21342 to branch-1 new 21cfff8 HBASE-20716 Unsafe access cleanup new 2f541f57 HBASE-21208 Bytes#toShort doesn't work without unsafe new 231e6d5 HBASE-20928 Rewrite calculation of midpoint in binarySearch functions to prevent overflow new f3f3798 HBASE-21164 reportForDuty should do backoff rather than retry The 4 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../apache/hadoop/hbase/filter/FuzzyRowFilter.java | 33 +- .../hadoop/hbase/io/encoding/RowIndexSeekerV1.java | 14 +- .../apache/hadoop/hbase/util/ByteBufferUtils.java | 543 ++--- .../java/org/apache/hadoop/hbase/util/Bytes.java | 220 ++--- .../org/apache/hadoop/hbase/util/RetryCounter.java | 10 + .../java/org/apache/hadoop/hbase/util/Sleeper.java | 31 +- .../org/apache/hadoop/hbase/util/UnsafeAccess.java | 267 ++ .../org/apache/hadoop/hbase/util/TestBytes.java| 45 ++ .../hadoop/hbase/io/hfile/HFileBlockIndex.java | 2 +- .../org/apache/hadoop/hbase/master/HMaster.java| 13 +- .../hadoop/hbase/regionserver/HRegionServer.java | 15 +- .../hbase/util/BoundedPriorityBlockingQueue.java | 2 +- .../TestRegionServerReportForDuty.java | 89 13 files changed, 1114 insertions(+), 170 deletions(-)
[hbase] 01/04: HBASE-20716 Unsafe access cleanup
This is an automated email from the ASF dual-hosted git repository.
apurtell pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 21cfff857dd6f241d345bebe74b31901e4b0efa5
Author: Sahil Aggarwal
AuthorDate: Tue Oct 9 22:41:36 2018 +0530
HBASE-20716 Unsafe access cleanup
Changes the bytes[] conversion done in Bytes and ByteBufferUtils.
Instead of doing check unsafe_aligned available everytime, choose
the best converter at startup.
Conflicts:
hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java
hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
Amending-Author: Andrew Purtell
---
.../apache/hadoop/hbase/filter/FuzzyRowFilter.java | 33 +-
.../apache/hadoop/hbase/util/ByteBufferUtils.java | 543 ++---
.../java/org/apache/hadoop/hbase/util/Bytes.java | 213 +---
.../org/apache/hadoop/hbase/util/UnsafeAccess.java | 267 ++
4 files changed, 918 insertions(+), 138 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
index d93d234..a2bcfb8 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
@@ -33,7 +33,6 @@ import
org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.BytesBytesPair;
import org.apache.hadoop.hbase.util.ByteStringer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.util.UnsafeAccess;
import org.apache.hadoop.hbase.util.UnsafeAvailChecker;
import com.google.common.annotations.VisibleForTesting;
@@ -342,19 +341,13 @@ public class FuzzyRowFilter extends FilterBase {
}
length = Math.min(length, fuzzyKeyBytes.length);
int numWords = length / Bytes.SIZEOF_LONG;
-int offsetAdj = offset + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET;
int j = numWords << 3; // numWords * SIZEOF_LONG;
for (int i = 0; i < j; i += Bytes.SIZEOF_LONG) {
-
- long fuzzyBytes =
- UnsafeAccess.theUnsafe.getLong(fuzzyKeyBytes,
UnsafeAccess.BYTE_ARRAY_BASE_OFFSET
- + (long) i);
- long fuzzyMeta =
- UnsafeAccess.theUnsafe.getLong(fuzzyKeyMeta,
UnsafeAccess.BYTE_ARRAY_BASE_OFFSET
- + (long) i);
- long rowValue = UnsafeAccess.theUnsafe.getLong(row, offsetAdj + (long)
i);
+ long fuzzyBytes = Bytes.toLong(fuzzyKeyBytes, i);
+ long fuzzyMeta = Bytes.toLong(fuzzyKeyMeta, i);
+ long rowValue = Bytes.toLong(row, offset + i);
if ((rowValue & fuzzyMeta) != (fuzzyBytes)) {
// We always return NEXT_EXISTS
return SatisfiesCode.NEXT_EXISTS;
@@ -364,13 +357,9 @@ public class FuzzyRowFilter extends FilterBase {
int off = j;
if (length - off >= Bytes.SIZEOF_INT) {
- int fuzzyBytes =
- UnsafeAccess.theUnsafe.getInt(fuzzyKeyBytes,
UnsafeAccess.BYTE_ARRAY_BASE_OFFSET
- + (long) off);
- int fuzzyMeta =
- UnsafeAccess.theUnsafe.getInt(fuzzyKeyMeta,
UnsafeAccess.BYTE_ARRAY_BASE_OFFSET
- + (long) off);
- int rowValue = UnsafeAccess.theUnsafe.getInt(row, offsetAdj + (long)
off);
+ int fuzzyBytes = Bytes.toInt(fuzzyKeyBytes, off);
+ int fuzzyMeta = Bytes.toInt(fuzzyKeyMeta, off);
+ int rowValue = Bytes.toInt(row, offset + off);
if ((rowValue & fuzzyMeta) != (fuzzyBytes)) {
// We always return NEXT_EXISTS
return SatisfiesCode.NEXT_EXISTS;
@@ -379,13 +368,9 @@ public class FuzzyRowFilter extends FilterBase {
}
if (length - off >= Bytes.SIZEOF_SHORT) {
- short fuzzyBytes =
- UnsafeAccess.theUnsafe.getShort(fuzzyKeyBytes,
UnsafeAccess.BYTE_ARRAY_BASE_OFFSET
- + (long) off);
- short fuzzyMeta =
- UnsafeAccess.theUnsafe.getShort(fuzzyKeyMeta,
UnsafeAccess.BYTE_ARRAY_BASE_OFFSET
- + (long) off);
- short rowValue = UnsafeAccess.theUnsafe.getShort(row, offsetAdj + (long)
off);
+ short fuzzyBytes = Bytes.toShort(fuzzyKeyBytes, off);
+ short fuzzyMeta = Bytes.toShort(fuzzyKeyMeta, off);
+ short rowValue = Bytes.toShort(row, offset + off);
if ((rowValue & fuzzyMeta) != (fuzzyBytes)) {
// We always return NEXT_EXISTS
// even if it does not (in this case getNextForFuzzyRule
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java
index b5b1d96..36fdec5 100644
---
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java
+++
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.jav
[hbase] 04/04: HBASE-21164 reportForDuty should do backoff rather than retry
This is an automated email from the ASF dual-hosted git repository.
apurtell pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit f3f3798575527df45961f4cbdb6d4c1d04cfb1e3
Author: Mingliang Liu
AuthorDate: Thu Sep 6 23:01:52 2018 -0700
HBASE-21164 reportForDuty should do backoff rather than retry
Remove unused methods from Sleeper (its ok, its @Private).
Remove notion of startTime from Sleeper handling (it is is unused).
Allow passing in how long to sleep so can maintain externally.
In HRS, use a RetryCounter to calculate backoff sleep time for when
reportForDuty is failing against a struggling Master.
Conflicts:
hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java
Amending-Author: Andrew Purtell
---
.../org/apache/hadoop/hbase/util/RetryCounter.java | 10 +++
.../java/org/apache/hadoop/hbase/util/Sleeper.java | 31 +++-
.../org/apache/hadoop/hbase/master/HMaster.java| 13 +++-
.../hadoop/hbase/regionserver/HRegionServer.java | 15 +++-
.../TestRegionServerReportForDuty.java | 89 ++
5 files changed, 132 insertions(+), 26 deletions(-)
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounter.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounter.java
index 73512fa..881dd36 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounter.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounter.java
@@ -174,4 +174,14 @@ public class RetryCounter {
public int getAttemptTimes() {
return attempts;
}
+
+ public long getBackoffTime() {
+return this.retryConfig.backoffPolicy.getBackoffTime(this.retryConfig,
getAttemptTimes());
+ }
+
+ public long getBackoffTimeAndIncrementAttempts() {
+long backoffTime = getBackoffTime();
+useRetry();
+return backoffTime;
+ }
}
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Sleeper.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Sleeper.java
index a60c571..a07ee9c 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Sleeper.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Sleeper.java
@@ -50,13 +50,6 @@ public class Sleeper {
}
/**
- * Sleep for period.
- */
- public void sleep() {
-sleep(System.currentTimeMillis());
- }
-
- /**
* If currently asleep, stops sleeping; if not asleep, will skip the next
* sleep cycle.
*/
@@ -68,28 +61,24 @@ public class Sleeper {
}
/**
- * Sleep for period adjusted by passed startTime
- * @param startTime Time some task started previous to now. Time to sleep
- * will be docked current time minus passed startTime.
+ * Sleep for period.
*/
- public void sleep(final long startTime) {
+ public void sleep() {
+sleep(this.period);
+ }
+
+ public void sleep(long sleepTime) {
if (this.stopper.isStopped()) {
return;
}
long now = System.currentTimeMillis();
-long waitTime = this.period - (now - startTime);
-if (waitTime > this.period) {
- LOG.warn("Calculated wait time > " + this.period +
-"; setting to this.period: " + System.currentTimeMillis() + ", " +
-startTime);
- waitTime = this.period;
-}
-while (waitTime > 0) {
+long currentSleepTime = sleepTime;
+while (currentSleepTime > 0) {
long woke = -1;
try {
synchronized (sleepLock) {
if (triggerWake) break;
- sleepLock.wait(waitTime);
+ sleepLock.wait(currentSleepTime);
}
woke = System.currentTimeMillis();
long slept = woke - now;
@@ -108,7 +97,7 @@ public class Sleeper {
}
// Recalculate waitTime.
woke = (woke == -1)? System.currentTimeMillis(): woke;
- waitTime = this.period - (woke - startTime);
+ currentSleepTime = this.period - (woke - now);
}
synchronized(sleepLock) {
triggerWake = false;
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index c2f99f4..d17e297 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -2656,7 +2656,18 @@ public class HMaster extends HRegionServer implements
MasterServices, Server {
stop("Stopped by " + Thread.currentThread().getName());
}
- void checkServiceStarted() throws ServerNotRunningYetException {
+ @Override
+ public void stop(String msg) {
+if (!isStopped()) {
+ super.stop(msg);
+ if (this.activeMasterManager != null)
[hbase] 03/04: HBASE-20928 Rewrite calculation of midpoint in binarySearch functions to prevent overflow
This is an automated email from the ASF dual-hosted git repository.
apurtell pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 231e6d56b569ca17fe620b1d29749dd211e1de02
Author: Saurabh Singh
AuthorDate: Fri Jan 11 12:14:18 2019 -0800
HBASE-20928 Rewrite calculation of midpoint in binarySearch functions to
prevent overflow
HBASE-20928 Rewrite calculation of midpoint - addendum (Xu Cang)
Signed-off-by: tedyu
Conflicts:
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java
hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
Amending-Author: Andrew Purtell
---
.../apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java | 14 +-
.../src/main/java/org/apache/hadoop/hbase/util/Bytes.java | 6 +++---
.../org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java | 2 +-
.../hadoop/hbase/util/BoundedPriorityBlockingQueue.java| 2 +-
4 files changed, 14 insertions(+), 10 deletions(-)
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java
index e72c685..f210fc5 100644
---
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java
+++
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java
@@ -154,15 +154,13 @@ public class RowIndexSeekerV1 extends
AbstractEncodedSeeker {
private int binarySearch(Cell seekCell, boolean seekBefore) {
int low = 0;
int high = rowNumber - 1;
-int mid = (low + high) >>> 1;
+int mid = low + ((high - low) >> 1);
int comp = 0;
SimpleMutableByteRange row = new SimpleMutableByteRange();
while (low <= high) {
- mid = (low + high) >>> 1;
+ mid = low + ((high - low) >> 1);
getRow(mid, row);
- comp = comparator.compareRows(row.getBytes(), row.getOffset(),
- row.getLength(), seekCell.getRowArray(), seekCell.getRowOffset(),
- seekCell.getRowLength());
+ comp = compareRows(row, seekCell);
if (comp < 0) {
low = mid + 1;
} else if (comp > 0) {
@@ -184,6 +182,12 @@ public class RowIndexSeekerV1 extends
AbstractEncodedSeeker {
}
}
+ private int compareRows(SimpleMutableByteRange row, Cell seekCell) {
+return comparator.compareRows(row.getBytes(), row.getOffset(),
+ row.getLength(), seekCell.getRowArray(), seekCell.getRowOffset(),
+ seekCell.getRowLength());
+ }
+
private void getRow(int index, SimpleMutableByteRange row) {
int offset = Bytes.toIntUnsafe(rowOffsets.array(), rowOffsets.arrayOffset()
+ (index << 2)); // index * Bytes.SIZEOF_INT
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
index 765f51b..22ade4b 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
@@ -2145,7 +2145,7 @@ public class Bytes implements Comparable {
int high = arr.length - 1;
while (low <= high) {
- int mid = (low+high) >>> 1;
+ int mid = low + ((high - low) >> 1);
// we have to compare in this order, because the comparator order
// has special logic when the 'left side' is a special key.
int cmp = comparator.compare(key, offset, length,
@@ -2182,7 +2182,7 @@ public class Bytes implements Comparable {
int high = arr.length - 1;
KeyValue.KeyOnlyKeyValue r = new KeyValue.KeyOnlyKeyValue();
while (low <= high) {
- int mid = (low+high) >>> 1;
+ int mid = low + ((high - low) >> 1);
// we have to compare in this order, because the comparator order
// has special logic when the 'left side' is a special key.
r.setKey(arr[mid], 0, arr[mid].length);
@@ -2357,7 +2357,7 @@ public class Bytes implements Comparable {
int high = toIndex - 1;
while (low <= high) {
- int mid = (low + high) >>> 1;
+ int mid = low + ((high - low) >> 1);
int midVal = a[mid] & 0xff;
if (midVal < unsignedKey) {
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
index da6f3b6..9148815 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
@@ -533,7 +533,7 @@ public class HFileBlockIndex {
// keys[low - 1] < key < keys[high + 1] while narrowing down the range.
KeyValue.KeyOnlyKeyValue nonRootIndexKV = new KeyValue.KeyOnlyKeyValue();
while (low <= high) {
-mid = (low + high) >>> 1;
+mid = low + ((high - low) >> 1);
// Midkey's offset re
[hbase] branch branch-1 updated: HBASE-21325 Force to terminate regionserver when abort hang in somewhere
This is an automated email from the ASF dual-hosted git repository.
apurtell pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new e7ff91f HBASE-21325 Force to terminate regionserver when abort hang
in somewhere
e7ff91f is described below
commit e7ff91f35ee9fbf9f66ef6e999a8155368d91753
Author: Guanghao Zhang
AuthorDate: Fri Oct 19 19:34:04 2018 +0800
HBASE-21325 Force to terminate regionserver when abort hang in somewhere
Conflicts:
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerAbortTimeout.java
Amending-Author: Andrew Purtell
---
.../hadoop/hbase/regionserver/HRegionServer.java | 39 +-
.../regionserver/TestRegionServerAbortTimeout.java | 137 +
2 files changed, 175 insertions(+), 1 deletion(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 6e5ce80..5f250c0 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -51,6 +51,8 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedMap;
+import java.util.Timer;
+import java.util.TimerTask;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
@@ -312,6 +314,11 @@ public class HRegionServer extends HasThread implements
// Go down hard. Used if file system becomes unavailable and also in
// debugging and unit tests.
private volatile boolean abortRequested;
+ public static final String ABORT_TIMEOUT =
"hbase.regionserver.abort.timeout";
+ // Default abort timeout is 1200 seconds for safe
+ private static final long DEFAULT_ABORT_TIMEOUT = 120;
+ // Will run this task when abort timeout
+ public static final String ABORT_TIMEOUT_TASK =
"hbase.regionserver.abort.timeout.task";
ConcurrentMap rowlocks = new ConcurrentHashMap();
@@ -1041,12 +1048,31 @@ public class HRegionServer extends HasThread implements
abort(prefix + t.getMessage(), t);
}
}
+
// Run shutdown.
if (mxBean != null) {
MBeanUtil.unregisterMBean(mxBean);
mxBean = null;
}
-if (this.leases != null) this.leases.closeAfterLeasesExpire();
+
+if (abortRequested) {
+ Timer abortMonitor = new Timer("Abort regionserver monitor", true);
+ TimerTask abortTimeoutTask = null;
+ try {
+abortTimeoutTask =
+Class.forName(conf.get(ABORT_TIMEOUT_TASK,
SystemExitWhenAbortTimeout.class.getName()))
+
.asSubclass(TimerTask.class).getDeclaredConstructor().newInstance();
+ } catch (Exception e) {
+LOG.warn("Initialize abort timeout task failed", e);
+ }
+ if (abortTimeoutTask != null) {
+abortMonitor.schedule(abortTimeoutTask, conf.getLong(ABORT_TIMEOUT,
DEFAULT_ABORT_TIMEOUT));
+ }
+}
+
+if (this.leases != null) {
+ this.leases.closeAfterLeasesExpire();
+}
if (this.splitLogWorker != null) {
splitLogWorker.stop();
}
@@ -3552,4 +3578,15 @@ public class HRegionServer extends HasThread implements
public void unassign(byte[] regionName) throws IOException {
clusterConnection.getAdmin().unassign(regionName, false);
}
+
+ /**
+ * Force to terminate region server when abort timeout.
+ */
+ private static class SystemExitWhenAbortTimeout extends TimerTask {
+@Override
+public void run() {
+ LOG.warn("Aborting region server timed out, terminate forcibly...");
+ System.exit(1);
+}
+ }
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerAbortTimeout.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerAbortTimeout.java
new file mode 100644
index 000..ed129c5
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerAbortTimeout.java
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANT
[hbase] branch branch-1 updated: HBASE-21679 Port HBASE-6028 (Start/Stop compactions at region server level) to branch-1
This is an automated email from the ASF dual-hosted git repository.
apurtell pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new d6a3e6b HBASE-21679 Port HBASE-6028 (Start/Stop compactions at region
server level) to branch-1
d6a3e6b is described below
commit d6a3e6b404a775edaf28cefb326e5186fd76b9dd
Author: Andrew Purtell
AuthorDate: Thu Jan 10 18:07:36 2019 -0800
HBASE-21679 Port HBASE-6028 (Start/Stop compactions at region server level)
to branch-1
HBASE-6028 Start/Stop compactions at region server level
Add switching on/off of compactions.
Switching off compactions will also interrupt any currently ongoing
compactions.
Adds a "compaction_switch" to hbase shell. Switching off compactions will
interrupt any currently ongoing compactions. State set from shell will be
lost on restart. To persist the changes across region servers modify
hbase.regionserver.compaction.enabled in hbase-site.xml and restart.
---
.../java/org/apache/hadoop/hbase/client/Admin.java | 13 +
.../org/apache/hadoop/hbase/client/HBaseAdmin.java | 47 +
hbase-common/src/main/resources/hbase-default.xml |7 +
.../hbase/protobuf/generated/AdminProtos.java | 1268 +---
hbase-protocol/src/main/protobuf/Admin.proto | 11 +
.../hbase/regionserver/CompactSplitThread.java | 149 ++-
.../hbase/regionserver/CompactionRequestor.java|5 +
.../apache/hadoop/hbase/regionserver/HRegion.java | 23 +-
.../hadoop/hbase/regionserver/RSRpcServices.java | 22 +
.../apache/hadoop/hbase/regionserver/Region.java |2 +-
.../hadoop/hbase/master/MockRegionServer.java |8 +
.../hadoop/hbase/regionserver/TestCompaction.java | 86 +-
.../replication/regionserver/TestReplicator.java |6 +
hbase-shell/src/main/ruby/hbase/admin.rb | 17 +-
hbase-shell/src/main/ruby/shell.rb |1 +
.../main/ruby/shell/commands/compaction_switch.rb | 53 +
src/main/asciidoc/_chapters/architecture.adoc |8 +
17 files changed, 1517 insertions(+), 209 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index 8f5e150..ea87457 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -634,6 +634,19 @@ public interface Admin extends Abortable, Closeable {
throws IOException;
/**
+ * Turn the compaction on or off. Disabling compactions will also interrupt
any currently ongoing
+ * compactions. It is ephemeral. This setting will be lost on restart of the
server. Compaction
+ * can also be enabled/disabled by modifying configuration
hbase.regionserver.compaction.enabled
+ * in hbase-site.xml.
+ *
+ * @param switchState Set to true to enable,
false to disable.
+ * @param serverNamesList list of region servers.
+ * @return Previous compaction states for region servers
+ */
+ Map compactionSwitch(boolean switchState, List
serverNamesList)
+ throws IOException;
+
+ /**
* Compact all regions on the region server
* @param sn the region server name
* @param major if it's major compaction
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index fafc9fe..7e624cc 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -83,6 +83,8 @@ import
org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
import
org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
import
org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
import
org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactRegionRequest;
+import
org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactionSwitchRequest;
+import
org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactionSwitchResponse;
import
org.apache.hadoop.hbase.protobuf.generated.AdminProtos.FlushRegionRequest;
import
org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
import
org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse;
@@ -2052,6 +2054,51 @@ public class HBaseAdmin implements Admin {
* {@inheritDoc}
*/
@Override
+ public Map compactionSwitch(boolean switchState,
List
+ serverNamesList) throws IOException {
+List serverList = new ArrayList<>();
+if (serverNamesList.isEmpty()) {
+ ClusterStatus status = getClusterStatus();
+ serverList.addAll(status.getServers());
+} else {
+ for (String regionServerName :
