This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
from 8b8c684bcc0 HBASE-27864 Reduce the Cardinality for
TestFuzzyRowFilterEndToEndLarge (#5237)
new e2d8db6dfc1 HBASE-27212 Implement a new table based replication queue
storage and make the minimum replication system work (#4672)
new f81bdebedba HBASE-27213 Add support for claim queue operation (#4708)
new 6d0311c1d96 HBASE-27214 Implement the new replication hfile/log
cleaner (#4722)
new 70441505451 HBASE-27215 Add support for sync replication (#4762)
new c01c8e45b47 HBASE-27392 Add a new procedure type for implementing some
global operations such as migration (#4803)
new 5f95a914b63 HBASE-27405 Fix the replication hfile/log cleaner report
that the replication table does not exist (#4811)
new 0d57ee147e2 HBASE-27218 Support rolling upgrading (#4808)
new ffad1ff727b HBASE-27217 Revisit the DumpReplicationQueues tool (#4810)
new dfb125f3b2b HBASE-27429 Add exponential retry backoff support for
MigrateReplicationQueueFromZkToTableProcedure
new 7c74f9e8c59 HBASE-27430 Should disable replication log cleaner when
migrating replication queue data (#4901)
new 000e64abda8 HBASE-27216 Revisit the ReplicationSyncUp tool (#4966)
new 650350a7491 HBASE-27623 Start a new ReplicationSyncUp after the
previous failed (#5150)
new 8fe691c705a HBASE-27775 Use a separate WAL provider for
hbase:replication table (#5157)
new 679dfcfd4fa HBASE-27274 Re-enable the disabled tests when implementing
HBASE-27212 (#5178)
new b34356202aa HBASE-27809 Attach move replication queue storage from
zookeeper to a separated HBase table design doc to git repo (#5197)
new 93ddd7060de HBASE-27516 Document the table based replication queue
storage in ref guide (#5203)
The 16 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
...e from zookeeper to a separated HBase table.pdf | Bin 0 -> 104629 bytes
.../org/apache/hadoop/hbase/util/JsonMapper.java | 4 +
.../apache/hadoop/hbase/zookeeper/ZNodePaths.java | 8 +-
.../hbase/procedure2/LockedResourceType.java | 3 +-
.../apache/hadoop/hbase/procedure2/Procedure.java | 15 +
.../hbase/procedure2/TimeoutExecutorThread.java | 10 +-
.../protobuf/server/master/MasterProcedure.proto | 32 +-
hbase-replication/pom.xml | 10 +
.../hbase/replication/ReplicationGroupOffset.java | 51 +-
...rStorageType.java => ReplicationQueueData.java} | 25 +-
.../hbase/replication/ReplicationQueueId.java | 141 ++++
.../hbase/replication/ReplicationQueueStorage.java | 194 +++--
.../replication/ReplicationStorageFactory.java | 66 +-
.../hadoop/hbase/replication/ReplicationUtils.java | 15 -
.../replication/TableReplicationQueueStorage.java | 617 ++++++++++++++
.../replication/ZKReplicationQueueStorage.java | 689 ---------------
.../ZKReplicationQueueStorageForMigration.java | 351 ++++++++
.../replication/TestReplicationStateBasic.java | 243 +-----
.../replication/TestReplicationStateZKImpl.java | 1 -
.../replication/TestZKReplicationQueueStorage.java | 470 +++++------
hbase-server/pom.xml | 6 +
.../org/apache/hadoop/hbase/master/HMaster.java | 57 +-
.../hbase/master/cleaner/FileCleanerDelegate.java | 2 +-
.../procedure/GlobalProcedureInterface.java} | 9 +-
.../procedure/{MetaQueue.java => GlobalQueue.java} | 12 +-
.../master/procedure/MasterProcedureScheduler.java | 119 ++-
.../hbase/master/procedure/SchemaLocking.java | 18 +-
.../master/procedure/ServerCrashProcedure.java | 45 +-
.../hadoop/hbase/master/region/MasterRegion.java | 2 +-
.../replication/AbstractPeerNoLockProcedure.java | 5 +-
.../hbase/master/replication/AddPeerProcedure.java | 21 +-
.../AssignReplicationQueuesProcedure.java | 241 ++++++
.../ClaimReplicationQueueRemoteProcedure.java | 76 +-
.../ClaimReplicationQueuesProcedure.java | 44 +-
...rateReplicationQueueFromZkToTableProcedure.java | 330 ++++++++
.../master/replication/ModifyPeerProcedure.java | 28 +-
.../OfflineTableReplicationQueueStorage.java | 382 +++++++++
.../master/replication/RemovePeerProcedure.java | 41 +-
.../master/replication/ReplicationPeerManager.java | 247 +++++-
.../TransitPeerSyncReplicationStateProcedure.java | 14 +
.../hadoop/hbase/regionserver/HRegionServer.java | 2 +-
.../MetricsRegionServerWrapperImpl.java | 12 +-
.../hbase/regionserver/wal/AbstractFSWAL.java | 6 +-
.../hbase/replication/ReplicationOffsetUtil.java | 47 +-
.../master/ReplicationHFileCleaner.java | 71 +-
.../replication/master/ReplicationLogCleaner.java | 233 +++--
.../master/ReplicationLogCleanerBarrier.java | 85 ++
.../ClaimReplicationQueueCallable.java | 17 +-
.../regionserver/DumpReplicationQueues.java | 222 +++--
.../regionserver/PeerProcedureHandler.java | 5 +-
.../regionserver/PeerProcedureHandlerImpl.java | 6 +-
.../regionserver/RecoveredReplicationSource.java | 138 +--
.../RecoveredReplicationSourceShipper.java | 64 +-
.../replication/regionserver/Replication.java | 2 +-
.../replication/regionserver/ReplicationLoad.java | 4 +-
.../regionserver/ReplicationSource.java | 72 +-
.../regionserver/ReplicationSourceFactory.java | 7 +-
.../regionserver/ReplicationSourceInterface.java | 26 +-
.../regionserver/ReplicationSourceLogQueue.java | 4 +-
.../regionserver/ReplicationSourceManager.java | 452 +++++-----
.../regionserver/ReplicationSourceShipper.java | 6 +-
.../ReplicationSourceWALActionListener.java | 5 -
.../regionserver/ReplicationSyncUp.java | 233 ++++-
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 4 +-
.../hadoop/hbase/util/hbck/ReplicationChecker.java | 47 +-
.../hadoop/hbase/wal/AbstractFSWALProvider.java | 93 ++
.../hbase/wal/LazyInitializedWALProvider.java | 108 +++
.../org/apache/hadoop/hbase/wal/WALFactory.java | 201 +++--
.../hbase/client/TestAsyncReplicationAdminApi.java | 10 +-
.../master/assignment/MockMasterServices.java | 2 +-
.../hbase/master/cleaner/TestLogsCleaner.java | 227 ++---
.../cleaner/TestReplicationHFileCleaner.java | 122 +--
.../procedure/TestMasterProcedureScheduler.java | 48 ++
.../replication/TestMigrateReplicationQueue.java | 126 +++
...rateReplicationQueueFromZkToTableProcedure.java | 253 ++++++
...icationQueueFromZkToTableProcedureRecovery.java | 128 +++
...tReplicationPeerManagerMigrateQueuesFromZk.java | 211 +++++
.../TestBulkLoadReplicationHFileRefs.java | 2 +-
...StartupWithLegacyRegionReplicationEndpoint.java | 35 +-
.../hbase/replication/ReplicationSourceDummy.java | 13 +-
.../TestAddToSerialReplicationPeer.java | 5 +-
.../replication/TestClaimReplicationQueue.java | 10 +-
.../replication/TestMultiSlaveReplication.java | 6 +-
...TestNamespaceReplicationWithBulkLoadedData.java | 4 +-
...java => TestRemovePeerProcedureWaitForSCP.java} | 69 +-
.../hbase/replication/TestReplicationBase.java | 2 +-
.../TestReplicationEmptyWALRecovery.java | 1 -
.../TestReplicationOffsetUtil.java} | 48 +-
.../replication/TestReplicationSyncUpTool.java | 214 +++--
.../replication/TestReplicationSyncUpToolBase.java | 6 +-
...estReplicationSyncUpToolWithBulkLoadedData.java | 89 +-
.../TestTableReplicationQueueStorage.java | 474 +++++++++++
.../replication/master/TestLogCleanerBarrier.java | 45 +-
.../master/TestReplicationLogCleaner.java | 386 +++++++++
.../regionserver/TestDumpReplicationQueues.java | 156 ++--
.../regionserver/TestReplicationSource.java | 82 +-
.../regionserver/TestReplicationSourceManager.java | 935 +++++----------------
.../TestReplicationSourceManagerZkImpl.java | 107 ---
.../regionserver/TestReplicationWALEdits.java | 147 ++++
.../regionserver/TestSerialReplicationChecker.java | 17 +-
.../regionserver/TestWALEntrySinkFilter.java | 23 -
.../TestHBaseFsckCleanReplicationBarriers.java | 20 +-
.../hadoop/hbase/util/TestHBaseFsckEncryption.java | 3 +
.../apache/hadoop/hbase/util/TestHBaseFsckMOB.java | 3 +
.../hbase/util/TestHBaseFsckReplication.java | 55 +-
.../apache/hadoop/hbase/wal/TestWALFactory.java | 30 +-
.../apache/hadoop/hbase/wal/TestWALMethods.java | 14 +
pom.xml | 7 +-
src/main/asciidoc/_chapters/ops_mgt.adoc | 70 +-
109 files changed, 7337 insertions(+), 3684 deletions(-)
create mode 100644 dev-support/design-docs/HBASE-27109 Move replication queue
storage from zookeeper to a separated HBase table.pdf
copy
hbase-server/src/test/java/org/apache/hadoop/hbase/replication/DummyReplicationEndpoint.java
=>
hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationGroupOffset.java
(56%)
copy
hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/{ReplicationPeerStorageType.java
=> ReplicationQueueData.java} (61%)
create mode 100644
hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueId.java
create mode 100644
hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableReplicationQueueStorage.java
delete mode 100644
hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java
create mode 100644
hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorageForMigration.java
copy hbase-server/src/main/java/org/apache/hadoop/hbase/{util/KeyRange.java =>
master/procedure/GlobalProcedureInterface.java} (83%)
copy
hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/{MetaQueue.java
=> GlobalQueue.java} (77%)
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AssignReplicationQueuesProcedure.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/MigrateReplicationQueueFromZkToTableProcedure.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/OfflineTableReplicationQueueStorage.java
copy
hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/terminal/TerminalSize.java
=>
hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationOffsetUtil.java
(50%)
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleanerBarrier.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/wal/LazyInitializedWALProvider.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestMigrateReplicationQueue.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestMigrateReplicationQueueFromZkToTableProcedure.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestMigrateReplicationQueueFromZkToTableProcedureRecovery.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestReplicationPeerManagerMigrateQueuesFromZk.java
copy
hbase-server/src/test/java/org/apache/hadoop/hbase/replication/{TestClaimReplicationQueue.java
=> TestRemovePeerProcedureWaitForSCP.java} (69%)
copy
hbase-server/src/test/java/org/apache/hadoop/hbase/{regionserver/TestObservedExceptionsInBatch.java
=> replication/TestReplicationOffsetUtil.java} (51%)
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestTableReplicationQueueStorage.java
copy
hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestServerAndLoad.java
=>
hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestLogCleanerBarrier.java
(54%)
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestReplicationLogCleaner.java
delete mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerZkImpl.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationWALEdits.java
