This is an automated email from the ASF dual-hosted git repository.
elserj pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git.
from 6818ec2 HBASE-26613 The logic of the method incrementIV in Encryption
class has problem (#3968)
new 6aaef897 HBASE-26064 Introduce a StoreFileTracker to abstract the
store file tracking logic
new 43b40e9 HBASE-25988 Store the store file list by a file (#3578)
new 6e05376 HBASE-26079 Use StoreFileTracker when splitting and merging
(#3617)
new 090b2fe HBASE-26224 Introduce a MigrationStoreFileTracker to support
migrating from different store file tracker implementations (#3656)
new 0ee1689 HBASE-26246 Persist the StoreFileTracker configurations to
TableDescriptor when creating table (#3666)
new 2052e80 HBASE-26248 Should find a suitable way to let users specify
the store file tracker implementation (#3665)
new 5ff0f98 HBASE-26264 Add more checks to prevent misconfiguration on
store file tracker (#3681)
new fc4f6d1 HBASE-26280 Use store file tracker when snapshoting (#3685)
new 06db852 HBASE-26326 CreateTableProcedure fails when
FileBasedStoreFileTracker… (#3721)
new e4e7cf8 HBASE-26386 Refactor StoreFileTracker implementations to
expose the set method (#3774)
new 08d1171 HBASE-26328 Clone snapshot doesn't load reference files into
FILE SFT impl (#3749)
new 8bec26e HBASE-26263 [Rolling Upgrading] Persist the StoreFileTracker
configurations to TableDescriptor for existing tables (#3700)
new a288365 HBASE-26271 Cleanup the broken store files under data
directory (#3786)
new d00b5fa HBASE-26454 CreateTableProcedure still relies on temp dir and
renames… (#3845)
new 771e552 HBASE-26286: Add support for specifying store file tracker
when restoring or cloning snapshot
new f16b7b1 HBASE-26265 Update ref guide to mention the new store file
tracker im… (#3942)
The 16 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/hadoop/hbase/client/Admin.java | 44 +-
.../hadoop/hbase/client/AdminOverAsyncAdmin.java | 7 +-
.../org/apache/hadoop/hbase/client/AsyncAdmin.java | 14 +-
.../hadoop/hbase/client/AsyncHBaseAdmin.java | 6 +-
.../hbase/client/ColumnFamilyDescriptor.java | 5 +
.../client/ColumnFamilyDescriptorBuilder.java | 6 +
.../hadoop/hbase/client/RawAsyncHBaseAdmin.java | 24 +-
.../src/main/protobuf/server/master/Master.proto | 1 +
.../protobuf/server/master/MasterProcedure.proto | 1 +
.../StoreFileTracker.proto} | 16 +-
.../java/org/apache/hadoop/hbase/io/HFileLink.java | 63 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 31 +-
.../hadoop/hbase/master/MasterRpcServices.java | 2 +-
.../assignment/MergeTableRegionsProcedure.java | 20 +-
.../assignment/SplitTableRegionProcedure.java | 40 +-
.../hbase/master/migrate/RollingUpgradeChore.java | 130 ++++
.../master/procedure/CloneSnapshotProcedure.java | 106 +--
.../master/procedure/CreateTableProcedure.java | 43 +-
.../master/procedure/DeleteTableProcedure.java | 95 +--
.../master/procedure/ModifyTableProcedure.java | 5 +
.../master/procedure/RestoreSnapshotProcedure.java | 7 +-
.../hbase/master/snapshot/SnapshotManager.java | 27 +-
.../hadoop/hbase/mob/DefaultMobStoreCompactor.java | 28 +-
.../hadoop/hbase/mob/DefaultMobStoreFlusher.java | 4 +-
.../regionserver/AbstractMultiFileWriter.java | 6 +-
.../hbase/regionserver/BrokenStoreFileCleaner.java | 202 ++++++
.../regionserver/CreateStoreFileWriterParams.java | 134 ++++
.../regionserver/DateTieredMultiFileWriter.java | 2 +-
.../hbase/regionserver/DateTieredStoreEngine.java | 5 +-
.../hbase/regionserver/DefaultStoreEngine.java | 5 +-
.../hbase/regionserver/DefaultStoreFlusher.java | 11 +-
.../hadoop/hbase/regionserver/HMobStore.java | 3 +-
.../hbase/regionserver/HRegionFileSystem.java | 50 +-
.../hadoop/hbase/regionserver/HRegionServer.java | 27 +
.../apache/hadoop/hbase/regionserver/HStore.java | 735 +++++----------------
.../hadoop/hbase/regionserver/StoreContext.java | 14 +
.../hadoop/hbase/regionserver/StoreEngine.java | 482 +++++++++++++-
.../hbase/regionserver/StoreFileManager.java | 9 +
.../hadoop/hbase/regionserver/StoreFlusher.java | 9 +-
.../hadoop/hbase/regionserver/StoreUtils.java | 37 +-
.../hbase/regionserver/StripeMultiFileWriter.java | 2 +-
.../hbase/regionserver/StripeStoreEngine.java | 9 +-
.../hbase/regionserver/StripeStoreFlusher.java | 9 +-
.../compactions/AbstractMultiOutputCompactor.java | 11 +-
.../hbase/regionserver/compactions/Compactor.java | 81 ++-
.../compactions/DateTieredCompactor.java | 6 +-
.../regionserver/compactions/DefaultCompactor.java | 25 +-
.../regionserver/compactions/StripeCompactor.java | 2 +-
.../storefiletracker/DefaultStoreFileTracker.java | 68 ++
.../FileBasedStoreFileTracker.java | 162 +++++
.../MigrateStoreFileTrackerProcedure.java | 48 ++
.../MigrationStoreFileTracker.java | 96 +++
.../storefiletracker/StoreFileListFile.java | 144 ++++
.../storefiletracker/StoreFileTracker.java | 97 +++
.../storefiletracker/StoreFileTrackerBase.java | 180 +++++
.../storefiletracker/StoreFileTrackerFactory.java | 351 ++++++++++
.../access/SnapshotScannerHDFSAclHelper.java | 4 +-
.../hbase/snapshot/RestoreSnapshotHelper.java | 104 ++-
.../hadoop/hbase/snapshot/SnapshotManifest.java | 42 +-
.../util/compaction/MajorCompactionRequest.java | 1 -
.../org/apache/hadoop/hbase/TestIOFencing.java | 12 +-
.../org/apache/hadoop/hbase/client/TestAdmin.java | 6 +
.../org/apache/hadoop/hbase/client/TestAdmin3.java | 6 +
.../hbase/client/TestAsyncTableAdminApi.java | 6 +
.../hbase/client/TestAsyncTableAdminApi3.java | 6 +
.../TestCloneSnapshotFromClientCustomSFT.java | 71 ++
...oneSnapshotFromClientCloneLinksAfterDelete.java | 4 +-
.../hbase/client/TestMobSnapshotFromClient.java | 7 +-
.../hbase/client/TestSnapshotFromClient.java | 30 +-
.../hadoop/hbase/master/TestMasterFileSystem.java | 29 +-
.../migrate/TestMigrateStoreFileTracker.java | 107 +++
.../procedure/MasterProcedureTestingUtility.java | 7 +
...=> TestCloneSnapshotProcedureFileBasedSFT.java} | 20 +-
.../master/procedure/TestCreateTableProcedure.java | 33 +
.../master/procedure/TestDeleteTableProcedure.java | 66 --
.../hadoop/hbase/mob/FaultyMobStoreCompactor.java | 3 +-
.../regionserver/TestBrokenStoreFileCleaner.java | 225 +++++++
.../regionserver/TestCacheOnWriteInSchema.java | 6 +-
.../hbase/regionserver/TestCompactorMemLeak.java | 4 +-
.../hbase/regionserver/TestDefaultStoreEngine.java | 6 +-
.../regionserver/TestDirectStoreSplitsMerges.java | 32 +-
.../hadoop/hbase/regionserver/TestHRegion.java | 4 +-
.../hadoop/hbase/regionserver/TestHStore.java | 33 +-
.../hadoop/hbase/regionserver/TestHStoreFile.java | 19 +-
.../regionserver/TestMergesSplitsAddToTracker.java | 262 ++++++++
.../TestRegionMergeTransactionOnCluster.java | 6 +-
.../regionserver/TestStoreFileRefresherChore.java | 3 +-
.../regionserver/TestStoreScannerClosure.java | 6 +-
.../hbase/regionserver/TestStripeStoreEngine.java | 3 +
.../compactions/TestDateTieredCompactor.java | 12 +-
.../compactions/TestStripeCompactionPolicy.java | 12 +-
.../compactions/TestStripeCompactor.java | 12 +-
.../TestChangeStoreFileTracker.java | 242 +++++++
.../TestMigrationStoreFileTracker.java | 194 ++++++
.../TestRegionWithFileBasedStoreFileTracker.java | 108 +++
.../storefiletracker/TestStoreFileTracker.java | 61 ++
.../TestStoreFileTrackerFactory.java | 110 +++
.../hbase/rsgroup/VerifyingRSGroupAdmin.java | 5 +-
.../hbase/snapshot/MobSnapshotTestingUtils.java | 74 ++-
.../hbase/snapshot/SnapshotTestingUtils.java | 16 +-
hbase-shell/src/main/ruby/hbase/admin.rb | 4 +-
hbase-shell/src/main/ruby/hbase_constants.rb | 1 +
.../src/main/ruby/shell/commands/clone_snapshot.rb | 6 +-
.../hadoop/hbase/thrift2/client/ThriftAdmin.java | 4 +-
.../asciidoc/_chapters/store_file_tracking.adoc | 145 ++++
src/main/asciidoc/book.adoc | 1 +
106 files changed, 4700 insertions(+), 1227 deletions(-)
copy hbase-protocol-shaded/src/main/protobuf/server/{rsgroup/RSGroup.proto =>
region/StoreFileTracker.proto} (78%)
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/master/migrate/RollingUpgradeChore.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/BrokenStoreFileCleaner.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CreateStoreFileWriterParams.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrateStoreFileTrackerProcedure.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClientCustomSFT.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/master/migrate/TestMigrateStoreFileTracker.java
copy
hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/{TestSCPWithoutMetaWithoutZKCoordinated.java
=> TestCloneSnapshotProcedureFileBasedSFT.java} (62%)
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBrokenStoreFileCleaner.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestChangeStoreFileTracker.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTrackerFactory.java
create mode 100644 src/main/asciidoc/_chapters/store_file_tracking.adoc
