This is an automated email from the ASF dual-hosted git repository.
apurtell pushed a change to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git.
from 64a2e9c HBASE-26871 shaded mapreduce and shaded byo-hadoop client
artifacts contain no classes (#4279)
new 073656b HBASE-26064 Introduce a StoreFileTracker to abstract the
store file tracking logic
new 6013510 HBASE-25988 Store the store file list by a file (#3578)
new d08857d HBASE-26079 Use StoreFileTracker when splitting and merging
(#3617)
new b7c1adb HBASE-26224 Introduce a MigrationStoreFileTracker to support
migrating from different store file tracker implementations (#3656)
new b80efea HBASE-26246 Persist the StoreFileTracker configurations to
TableDescriptor when creating table (#3666)
new 864fb9a HBASE-26248 Should find a suitable way to let users specify
the store file tracker implementation (#3665)
new 9ed3f3f HBASE-26264 Add more checks to prevent misconfiguration on
store file tracker (#3681)
new 4ae866f HBASE-26280 Use store file tracker when snapshoting (#3685)
new 4b493b2 HBASE-26326 CreateTableProcedure fails when
FileBasedStoreFileTracker… (#3721)
new 08d108d HBASE-26386 Refactor StoreFileTracker implementations to
expose the set method (#3774)
new b5a3225 HBASE-26328 Clone snapshot doesn't load reference files into
FILE SFT impl (#3749)
new baac21b HBASE-26263 [Rolling Upgrading] Persist the StoreFileTracker
configurations to TableDescriptor for existing tables (#3700)
new d6a409a HBASE-26271 Cleanup the broken store files under data
directory (#3786)
new 6a794e3 HBASE-26454 CreateTableProcedure still relies on temp dir and
renames… (#3845)
new 16ad5f7 HBASE-26286: Add support for specifying store file tracker
when restoring or cloning snapshot
new 4776c88 HBASE-26265 Update ref guide to mention the new store file
tracker im… (#3942)
new 45d7f31 HBASE-26585 Add SFT configuration to META table descriptor
when creating META (#3998)
new 5ffa831 HBASE-26639 The implementation of
TestMergesSplitsAddToTracker is problematic (#4010)
new 90981be HBASE-26586 Should not rely on the global config when setting
SFT implementation for a table while upgrading (#4006)
new af921c3 HBASE-26654 ModifyTableDescriptorProcedure shoud load
TableDescriptor while executing (#4034)
new 9435769 HBASE-26674 Should modify filesCompacting under
storeWriteLock (#4040)
new 520b7e7 HBASE-26675 Data race on Compactor.writer (#4035)
new 46c10f7 HBASE-26700 The way we bypass broken track file is not enough
in StoreFileListFile (#4055)
new f0e1bc8 HBASE-26690 Modify FSTableDescriptors to not rely on renaming
when writing TableDescriptor (#4054)
new 6c43e62 HBASE-26587 Introduce a new Admin API to change SFT
implementation (#4030) (#4080)
new 4c0786e HBASE-26673 Implement a shell command for change SFT
implementation (#4113)
new c76c083 HBASE-26640 Reimplement master local region initialization to
better work with SFT (#4111)
new 4f9fbd8 HBASE-26707: Reduce number of renames during bulkload (#4066)
(#4122)
new 9c8d1e4 HBASE-26611 Changing SFT implementation on disabled table is
dangerous (#4082)
new 6a4b242 HBASE-26837 Set SFT config when creating TableDescriptor in
TestCloneSnapshotProcedure (#4226)
new 2bbddfe HBASE-26881 Backport HBASE-25368 to branch-2 (#4267)
new 6902cb2 HBASE-26826 Backport StoreFileTracker (HBASE-26067,
HBASE-26584, and others) to branch-2.5
The 32 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/hadoop/hbase/client/Admin.java | 88 ++-
.../org/apache/hadoop/hbase/client/AsyncAdmin.java | 30 +-
.../hadoop/hbase/client/AsyncHBaseAdmin.java | 17 +-
.../hbase/client/ColumnFamilyDescriptor.java | 5 +
.../client/ColumnFamilyDescriptorBuilder.java | 6 +
.../hbase/client/ConnectionImplementation.java | 18 +
.../org/apache/hadoop/hbase/client/HBaseAdmin.java | 141 +++-
.../hadoop/hbase/client/RawAsyncHBaseAdmin.java | 201 ++++--
.../org/apache/hadoop/hbase/client/RegionInfo.java | 20 +-
.../hbase/client/ShortCircuitMasterConnection.java | 16 +
.../hbase/client/TableDescriptorBuilder.java | 23 +-
.../hbase/shaded/protobuf/RequestConverter.java | 28 +-
.../IntegrationTestFileBasedSFTBulkLoad.java | 109 +++
.../hadoop/hbase/regionserver/CompactionTool.java | 2 +-
.../src/main/protobuf/Master.proto | 30 +
.../src/main/protobuf/MasterProcedure.proto | 27 +
.../src/main/protobuf/StoreFileTracker.proto | 36 +
.../hadoop/hbase/coprocessor/MasterObserver.java | 52 ++
.../java/org/apache/hadoop/hbase/io/HFileLink.java | 63 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 89 ++-
.../hadoop/hbase/master/MasterCoprocessorHost.java | 49 ++
.../hadoop/hbase/master/MasterRpcServices.java | 32 +-
.../apache/hadoop/hbase/master/MasterServices.java | 19 +
.../assignment/MergeTableRegionsProcedure.java | 20 +-
.../assignment/SplitTableRegionProcedure.java | 40 +-
.../hbase/master/migrate/RollingUpgradeChore.java | 130 ++++
.../master/procedure/CloneSnapshotProcedure.java | 106 +--
.../master/procedure/CreateTableProcedure.java | 45 +-
.../master/procedure/DeleteTableProcedure.java | 115 ++--
.../procedure/ModifyTableDescriptorProcedure.java | 177 +++++
.../master/procedure/ModifyTableProcedure.java | 5 +
.../master/procedure/RestoreSnapshotProcedure.java | 7 +-
.../hadoop/hbase/master/region/MasterRegion.java | 141 +++-
.../hbase/master/region/MasterRegionFactory.java | 22 +-
.../hbase/master/snapshot/SnapshotManager.java | 27 +-
.../hadoop/hbase/mob/DefaultMobStoreCompactor.java | 39 +-
.../hadoop/hbase/mob/DefaultMobStoreFlusher.java | 4 +-
.../hbase/regionserver/AbstractMemStore.java | 3 +-
.../regionserver/AbstractMultiFileWriter.java | 6 +-
.../hbase/regionserver/BrokenStoreFileCleaner.java | 202 ++++++
.../hbase/regionserver/CompactingMemStore.java | 2 +-
.../regionserver/CreateStoreFileWriterParams.java | 134 ++++
.../regionserver/DateTieredMultiFileWriter.java | 2 +-
.../hbase/regionserver/DateTieredStoreEngine.java | 5 +-
.../hbase/regionserver/DefaultStoreEngine.java | 5 +-
.../hbase/regionserver/DefaultStoreFlusher.java | 11 +-
.../hadoop/hbase/regionserver/HMobStore.java | 3 +-
.../apache/hadoop/hbase/regionserver/HRegion.java | 19 +-
.../hbase/regionserver/HRegionFileSystem.java | 54 +-
.../hadoop/hbase/regionserver/HRegionServer.java | 27 +
.../apache/hadoop/hbase/regionserver/HStore.java | 735 +++++----------------
.../hbase/regionserver/SecureBulkLoadManager.java | 64 +-
.../hadoop/hbase/regionserver/StoreContext.java | 14 +
.../hadoop/hbase/regionserver/StoreEngine.java | 484 +++++++++++++-
.../hbase/regionserver/StoreFileManager.java | 9 +
.../hadoop/hbase/regionserver/StoreFlusher.java | 9 +-
.../hadoop/hbase/regionserver/StoreUtils.java | 37 +-
.../hbase/regionserver/StripeMultiFileWriter.java | 2 +-
.../hbase/regionserver/StripeStoreEngine.java | 9 +-
.../hbase/regionserver/StripeStoreFlusher.java | 9 +-
.../compactions/AbstractMultiOutputCompactor.java | 11 +-
.../hbase/regionserver/compactions/Compactor.java | 83 ++-
.../compactions/DateTieredCompactor.java | 6 +-
.../regionserver/compactions/DefaultCompactor.java | 31 +-
.../regionserver/compactions/StripeCompactor.java | 2 +-
.../storefiletracker/DefaultStoreFileTracker.java | 68 ++
.../FileBasedStoreFileTracker.java | 162 +++++
.../InitializeStoreFileTrackerProcedure.java | 53 ++
.../MigrationStoreFileTracker.java | 96 +++
...odifyColumnFamilyStoreFileTrackerProcedure.java | 107 +++
.../ModifyStoreFileTrackerProcedure.java | 271 ++++++++
.../ModifyTableStoreFileTrackerProcedure.java | 71 ++
.../storefiletracker/StoreFileListFile.java | 169 +++++
.../storefiletracker/StoreFileTracker.java | 97 +++
.../storefiletracker/StoreFileTrackerBase.java | 180 +++++
.../storefiletracker/StoreFileTrackerFactory.java | 191 ++++++
.../StoreFileTrackerValidationUtils.java | 226 +++++++
.../hbase/security/access/AccessController.java | 20 +-
.../access/SnapshotScannerHDFSAclHelper.java | 4 +-
.../hbase/snapshot/RestoreSnapshotHelper.java | 102 ++-
.../hadoop/hbase/snapshot/SnapshotManifest.java | 42 +-
.../hadoop/hbase/util/FSTableDescriptors.java | 388 +++++------
.../util/compaction/MajorCompactionRequest.java | 1 -
.../org/apache/hadoop/hbase/TestIOFencing.java | 12 +-
.../org/apache/hadoop/hbase/client/TestAdmin.java | 6 +
.../org/apache/hadoop/hbase/client/TestAdmin1.java | 19 +
.../org/apache/hadoop/hbase/client/TestAdmin2.java | 9 +-
.../org/apache/hadoop/hbase/client/TestAdmin3.java | 163 +++++
.../TestAsyncAdminModifyStoreFileTracker.java | 197 ++++++
.../hbase/client/TestAsyncTableAdminApi.java | 7 +
.../hbase/client/TestAsyncTableAdminApi3.java | 8 +-
.../TestCloneSnapshotFromClientCustomSFT.java | 72 ++
...oneSnapshotFromClientCloneLinksAfterDelete.java | 4 +-
.../hbase/client/TestMobSnapshotFromClient.java | 7 +-
.../hbase/client/TestSnapshotFromClient.java | 34 +-
.../hbase/master/MockNoopMasterServices.java | 12 +
.../hadoop/hbase/master/TestMasterFileSystem.java | 29 +-
.../TestMasterFileSystemWithStoreFileTracking.java | 69 ++
.../migrate/TestInitializeStoreFileTracker.java | 119 ++++
.../procedure/MasterProcedureTestingUtility.java | 7 +
.../procedure/TestCloneSnapshotProcedure.java | 8 +-
.../TestCloneSnapshotProcedureFileBasedSFT.java | 42 ++
.../master/procedure/TestCreateTableProcedure.java | 33 +
.../master/procedure/TestDeleteTableProcedure.java | 66 --
.../hbase/master/region/MasterRegionTestBase.java | 16 +-
.../region/TestChangeSFTForMasterRegion.java | 86 +++
.../master/region/TestMasterRegionInitialize.java | 117 ++++
.../region/TestMasterRegionOnTwoFileSystems.java | 9 +-
.../regionserver/TestBrokenStoreFileCleaner.java | 226 +++++++
.../hadoop/hbase/regionserver/TestBulkLoad.java | 4 +
.../hbase/regionserver/TestBulkloadBase.java | 39 +-
.../regionserver/TestCacheOnWriteInSchema.java | 6 +-
.../regionserver/TestCompactionAfterBulkLoad.java | 4 +
.../hbase/regionserver/TestCompactorMemLeak.java | 4 +-
.../hbase/regionserver/TestDefaultStoreEngine.java | 10 +-
.../regionserver/TestDirectStoreSplitsMerges.java | 34 +-
.../hadoop/hbase/regionserver/TestHRegion.java | 4 +-
.../hadoop/hbase/regionserver/TestHStore.java | 33 +-
.../hadoop/hbase/regionserver/TestHStoreFile.java | 23 +-
.../regionserver/TestMergesSplitsAddToTracker.java | 274 ++++++++
.../TestRegionMergeTransactionOnCluster.java | 6 +-
.../regionserver/TestSecureBulkLoadManager.java | 39 +-
.../regionserver/TestSecureBulkloadListener.java | 267 ++++++++
.../regionserver/TestStoreFileRefresherChore.java | 3 +-
.../regionserver/TestStoreScannerClosure.java | 8 +-
.../hbase/regionserver/TestStripeStoreEngine.java | 7 +-
.../compactions/TestDateTieredCompactor.java | 17 +-
.../compactions/TestStripeCompactionPolicy.java | 12 +-
.../compactions/TestStripeCompactor.java | 17 +-
.../storefiletracker/StoreFileTrackerForTest.java | 72 ++
.../TestChangeStoreFileTracker.java | 264 ++++++++
.../TestMigrationStoreFileTracker.java | 194 ++++++
.../TestRegionWithFileBasedStoreFileTracker.java | 108 +++
.../storefiletracker/TestStoreFileListFile.java | 165 +++++
.../TestStoreFileTrackerFactory.java | 58 ++
.../TestStoreFileTrackerValidationUtils.java | 88 +++
.../hbase/snapshot/MobSnapshotTestingUtils.java | 71 +-
.../hbase/snapshot/SnapshotTestingUtils.java | 16 +-
.../hadoop/hbase/util/TestFSTableDescriptors.java | 307 +++++----
hbase-shell/src/main/ruby/hbase/admin.rb | 15 +-
hbase-shell/src/main/ruby/hbase_constants.rb | 1 +
hbase-shell/src/main/ruby/shell.rb | 9 +
.../src/main/ruby/shell/commands/change_sft.rb | 50 ++
.../src/main/ruby/shell/commands/change_sft_all.rb | 58 ++
.../src/main/ruby/shell/commands/clone_snapshot.rb | 6 +-
.../hadoop/hbase/client/TestChangeSftShell.java | 39 +-
.../src/test/ruby/shell/sftchange_shell_test.rb | 56 ++
.../hadoop/hbase/thrift2/client/ThriftAdmin.java | 18 +-
.../asciidoc/_chapters/store_file_tracking.adoc | 145 ++++
src/main/asciidoc/book.adoc | 1 +
150 files changed, 8316 insertions(+), 1818 deletions(-)
create mode 100644
hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestFileBasedSFTBulkLoad.java
create mode 100644
hbase-protocol-shaded/src/main/protobuf/StoreFileTracker.proto
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/master/migrate/RollingUpgradeChore.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableDescriptorProcedure.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/BrokenStoreFileCleaner.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CreateStoreFileWriterParams.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/InitializeStoreFileTrackerProcedure.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/ModifyColumnFamilyStoreFileTrackerProcedure.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/ModifyStoreFileTrackerProcedure.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/ModifyTableStoreFileTrackerProcedure.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerValidationUtils.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAdminModifyStoreFileTracker.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClientCustomSFT.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystemWithStoreFileTracking.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/master/migrate/TestInitializeStoreFileTracker.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedureFileBasedSFT.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/master/region/TestChangeSFTForMasterRegion.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/master/region/TestMasterRegionInitialize.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBrokenStoreFileCleaner.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSecureBulkloadListener.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerForTest.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestChangeStoreFileTracker.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileListFile.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTrackerFactory.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTrackerValidationUtils.java
create mode 100644 hbase-shell/src/main/ruby/shell/commands/change_sft.rb
create mode 100644 hbase-shell/src/main/ruby/shell/commands/change_sft_all.rb
copy
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotFromClient.java
=>
hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestChangeSftShell.java
(52%)
create mode 100644 hbase-shell/src/test/ruby/shell/sftchange_shell_test.rb
create mode 100644 src/main/asciidoc/_chapters/store_file_tracking.adoc
