This is an automated email from the ASF dual-hosted git repository. elserj pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git.
from 6818ec2 HBASE-26613 The logic of the method incrementIV in Encryption class has problem (#3968) new 6aaef897 HBASE-26064 Introduce a StoreFileTracker to abstract the store file tracking logic new 43b40e9 HBASE-25988 Store the store file list by a file (#3578) new 6e05376 HBASE-26079 Use StoreFileTracker when splitting and merging (#3617) new 090b2fe HBASE-26224 Introduce a MigrationStoreFileTracker to support migrating from different store file tracker implementations (#3656) new 0ee1689 HBASE-26246 Persist the StoreFileTracker configurations to TableDescriptor when creating table (#3666) new 2052e80 HBASE-26248 Should find a suitable way to let users specify the store file tracker implementation (#3665) new 5ff0f98 HBASE-26264 Add more checks to prevent misconfiguration on store file tracker (#3681) new fc4f6d1 HBASE-26280 Use store file tracker when snapshoting (#3685) new 06db852 HBASE-26326 CreateTableProcedure fails when FileBasedStoreFileTracker… (#3721) new e4e7cf8 HBASE-26386 Refactor StoreFileTracker implementations to expose the set method (#3774) new 08d1171 HBASE-26328 Clone snapshot doesn't load reference files into FILE SFT impl (#3749) new 8bec26e HBASE-26263 [Rolling Upgrading] Persist the StoreFileTracker configurations to TableDescriptor for existing tables (#3700) new a288365 HBASE-26271 Cleanup the broken store files under data directory (#3786) new d00b5fa HBASE-26454 CreateTableProcedure still relies on temp dir and renames… (#3845) new 771e552 HBASE-26286: Add support for specifying store file tracker when restoring or cloning snapshot new f16b7b1 HBASE-26265 Update ref guide to mention the new store file tracker im… (#3942) The 16 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../java/org/apache/hadoop/hbase/client/Admin.java | 44 +- .../hadoop/hbase/client/AdminOverAsyncAdmin.java | 7 +- .../org/apache/hadoop/hbase/client/AsyncAdmin.java | 14 +- .../hadoop/hbase/client/AsyncHBaseAdmin.java | 6 +- .../hbase/client/ColumnFamilyDescriptor.java | 5 + .../client/ColumnFamilyDescriptorBuilder.java | 6 + .../hadoop/hbase/client/RawAsyncHBaseAdmin.java | 24 +- .../src/main/protobuf/server/master/Master.proto | 1 + .../protobuf/server/master/MasterProcedure.proto | 1 + .../StoreFileTracker.proto} | 16 +- .../java/org/apache/hadoop/hbase/io/HFileLink.java | 63 +- .../org/apache/hadoop/hbase/master/HMaster.java | 31 +- .../hadoop/hbase/master/MasterRpcServices.java | 2 +- .../assignment/MergeTableRegionsProcedure.java | 20 +- .../assignment/SplitTableRegionProcedure.java | 40 +- .../hbase/master/migrate/RollingUpgradeChore.java | 130 ++++ .../master/procedure/CloneSnapshotProcedure.java | 106 +-- .../master/procedure/CreateTableProcedure.java | 43 +- .../master/procedure/DeleteTableProcedure.java | 95 +-- .../master/procedure/ModifyTableProcedure.java | 5 + .../master/procedure/RestoreSnapshotProcedure.java | 7 +- .../hbase/master/snapshot/SnapshotManager.java | 27 +- .../hadoop/hbase/mob/DefaultMobStoreCompactor.java | 28 +- .../hadoop/hbase/mob/DefaultMobStoreFlusher.java | 4 +- .../regionserver/AbstractMultiFileWriter.java | 6 +- .../hbase/regionserver/BrokenStoreFileCleaner.java | 202 ++++++ .../regionserver/CreateStoreFileWriterParams.java | 134 ++++ .../regionserver/DateTieredMultiFileWriter.java | 2 +- .../hbase/regionserver/DateTieredStoreEngine.java | 5 +- .../hbase/regionserver/DefaultStoreEngine.java | 5 +- .../hbase/regionserver/DefaultStoreFlusher.java | 11 +- .../hadoop/hbase/regionserver/HMobStore.java | 3 +- .../hbase/regionserver/HRegionFileSystem.java | 50 +- .../hadoop/hbase/regionserver/HRegionServer.java | 27 + .../apache/hadoop/hbase/regionserver/HStore.java | 735 +++++---------------- .../hadoop/hbase/regionserver/StoreContext.java | 14 + .../hadoop/hbase/regionserver/StoreEngine.java | 482 +++++++++++++- .../hbase/regionserver/StoreFileManager.java | 9 + .../hadoop/hbase/regionserver/StoreFlusher.java | 9 +- .../hadoop/hbase/regionserver/StoreUtils.java | 37 +- .../hbase/regionserver/StripeMultiFileWriter.java | 2 +- .../hbase/regionserver/StripeStoreEngine.java | 9 +- .../hbase/regionserver/StripeStoreFlusher.java | 9 +- .../compactions/AbstractMultiOutputCompactor.java | 11 +- .../hbase/regionserver/compactions/Compactor.java | 81 ++- .../compactions/DateTieredCompactor.java | 6 +- .../regionserver/compactions/DefaultCompactor.java | 25 +- .../regionserver/compactions/StripeCompactor.java | 2 +- .../storefiletracker/DefaultStoreFileTracker.java | 68 ++ .../FileBasedStoreFileTracker.java | 162 +++++ .../MigrateStoreFileTrackerProcedure.java | 48 ++ .../MigrationStoreFileTracker.java | 96 +++ .../storefiletracker/StoreFileListFile.java | 144 ++++ .../storefiletracker/StoreFileTracker.java | 97 +++ .../storefiletracker/StoreFileTrackerBase.java | 180 +++++ .../storefiletracker/StoreFileTrackerFactory.java | 351 ++++++++++ .../access/SnapshotScannerHDFSAclHelper.java | 4 +- .../hbase/snapshot/RestoreSnapshotHelper.java | 104 ++- .../hadoop/hbase/snapshot/SnapshotManifest.java | 42 +- .../util/compaction/MajorCompactionRequest.java | 1 - .../org/apache/hadoop/hbase/TestIOFencing.java | 12 +- .../org/apache/hadoop/hbase/client/TestAdmin.java | 6 + .../org/apache/hadoop/hbase/client/TestAdmin3.java | 6 + .../hbase/client/TestAsyncTableAdminApi.java | 6 + .../hbase/client/TestAsyncTableAdminApi3.java | 6 + .../TestCloneSnapshotFromClientCustomSFT.java | 71 ++ ...oneSnapshotFromClientCloneLinksAfterDelete.java | 4 +- .../hbase/client/TestMobSnapshotFromClient.java | 7 +- .../hbase/client/TestSnapshotFromClient.java | 30 +- .../hadoop/hbase/master/TestMasterFileSystem.java | 29 +- .../migrate/TestMigrateStoreFileTracker.java | 107 +++ .../procedure/MasterProcedureTestingUtility.java | 7 + ...=> TestCloneSnapshotProcedureFileBasedSFT.java} | 20 +- .../master/procedure/TestCreateTableProcedure.java | 33 + .../master/procedure/TestDeleteTableProcedure.java | 66 -- .../hadoop/hbase/mob/FaultyMobStoreCompactor.java | 3 +- .../regionserver/TestBrokenStoreFileCleaner.java | 225 +++++++ .../regionserver/TestCacheOnWriteInSchema.java | 6 +- .../hbase/regionserver/TestCompactorMemLeak.java | 4 +- .../hbase/regionserver/TestDefaultStoreEngine.java | 6 +- .../regionserver/TestDirectStoreSplitsMerges.java | 32 +- .../hadoop/hbase/regionserver/TestHRegion.java | 4 +- .../hadoop/hbase/regionserver/TestHStore.java | 33 +- .../hadoop/hbase/regionserver/TestHStoreFile.java | 19 +- .../regionserver/TestMergesSplitsAddToTracker.java | 262 ++++++++ .../TestRegionMergeTransactionOnCluster.java | 6 +- .../regionserver/TestStoreFileRefresherChore.java | 3 +- .../regionserver/TestStoreScannerClosure.java | 6 +- .../hbase/regionserver/TestStripeStoreEngine.java | 3 + .../compactions/TestDateTieredCompactor.java | 12 +- .../compactions/TestStripeCompactionPolicy.java | 12 +- .../compactions/TestStripeCompactor.java | 12 +- .../TestChangeStoreFileTracker.java | 242 +++++++ .../TestMigrationStoreFileTracker.java | 194 ++++++ .../TestRegionWithFileBasedStoreFileTracker.java | 108 +++ .../storefiletracker/TestStoreFileTracker.java | 61 ++ .../TestStoreFileTrackerFactory.java | 110 +++ .../hbase/rsgroup/VerifyingRSGroupAdmin.java | 5 +- .../hbase/snapshot/MobSnapshotTestingUtils.java | 74 ++- .../hbase/snapshot/SnapshotTestingUtils.java | 16 +- hbase-shell/src/main/ruby/hbase/admin.rb | 4 +- hbase-shell/src/main/ruby/hbase_constants.rb | 1 + .../src/main/ruby/shell/commands/clone_snapshot.rb | 6 +- .../hadoop/hbase/thrift2/client/ThriftAdmin.java | 4 +- .../asciidoc/_chapters/store_file_tracking.adoc | 145 ++++ src/main/asciidoc/book.adoc | 1 + 106 files changed, 4700 insertions(+), 1227 deletions(-) copy hbase-protocol-shaded/src/main/protobuf/server/{rsgroup/RSGroup.proto => region/StoreFileTracker.proto} (78%) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/migrate/RollingUpgradeChore.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/BrokenStoreFileCleaner.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CreateStoreFileWriterParams.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrateStoreFileTrackerProcedure.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClientCustomSFT.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/migrate/TestMigrateStoreFileTracker.java copy hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/{TestSCPWithoutMetaWithoutZKCoordinated.java => TestCloneSnapshotProcedureFileBasedSFT.java} (62%) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBrokenStoreFileCleaner.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestChangeStoreFileTracker.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTrackerFactory.java create mode 100644 src/main/asciidoc/_chapters/store_file_tracking.adoc