This is an automated email from the ASF dual-hosted git repository.
openinx pushed a change to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git.
from 974e108 HBASE-22617 Recovered WAL directories not getting cleaned up
(#330)
add c19592a HBASE-21916 Abstract an ByteBuffAllocator to allocate/free
ByteBuffer in ByteBufferPool
add 259f861 HBASE-21917 Make the HFileBlock#validateChecksum can accept
ByteBuff as an input.
add 287e014 HBASE-21917 Make the HFileBlock#validateChecksum can accept
ByteBuff as an input. (addendum)
add f97f6e3 HBASE-22016 Rewrite the block reading methods by using
hbase.nio.ByteBuff
add 0d294d2 HBASE-22005 Use ByteBuff's refcnt to track the life cycle of
data block
add 081dc03 HBASE-22127 Ensure that the block cached in the LRUBlockCache
offheap is allocated from heap
add bd8260c HBASE-22159 ByteBufferIOEngine should support write off-heap
ByteBuff to the bufferArray
add 92fc520 HBASE-20894 Use proto for BucketCache persistence
add fc46013 HBASE-21957 Unify refCount of BucketEntry and refCount of
hbase.nio.ByteBuff into one
add bb40caf HBASE-22211 Remove the returnBlock method because we can just
call HFileBlock#release directly
add 39a6a35 HBASE-21937 Make the Compression#decompress can accept
ByteBuff as input
add b3a01b5 HBASE-22122 Change to release mob hfile's block after rpc
server shipped response to client
add d7009d2 HBASE-21921 Notify users if the ByteBufAllocator is always
allocating ByteBuffers from heap which means the increacing GC pressure
add 02c528a HBASE-22090 The HFileBlock#CacheableDeserializer should pass
ByteBuffAllocator to the newly created HFileBlock
add b003607 HBASE-22412 Improve the metrics in ByteBuffAllocator
add 9e5fc2b HBASE-22435 Add a UT to address the HFileBlock#heapSize() in
TestHeapSize
add 1f5cf41 HBASE-22422 Retain an ByteBuff with refCnt=0 when getBlock
from LRUCache (#242)
add 915d7b2 HBASE-22463 Some paths in HFileScannerImpl did not consider
block#release which will exhaust the ByteBuffAllocator (#257)
add 4dd728d HBASE-22483 It's better to use 65KB as the default buffer
size in ByteBuffAllocator (#279)
add 5ff317f HBASE-22504 Optimize the MultiByteBuff#get(ByteBuffer,
offset, len)
add 3199175 HBASE-22491 Separate the heap HFileBlock and offheap
HFileBlock because the heap block won't need refCnt and save into prevBlocks
list before shipping (#268)
add 4f1b5ab HBASE-22531 The HFileReaderImpl#shouldUseHeap return the
incorrect true when disabled BlockCache (#304)
add d34c4a0 HBASE-22598 Deprecated the
hbase.ipc.server.reservoir.initial.buffer.size &
hbase.ipc.server.reservoir.initial.max for HBase2.x compatibility (#318)
add af0e23c HBASE-22547 Align the config keys and add document for
offheap read in HBase Book. (#301)
new 6a3aa78 HBASE-22612 Address the final overview reviewing comments of
HBASE-21879
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/hadoop/hbase/ipc/CellBlockBuilder.java | 9 +-
hbase-common/pom.xml | 4 +
.../apache/hadoop/hbase/io/ByteBuffAllocator.java | 393 +++++++++++
.../hbase/io/ByteBufferListOutputStream.java | 40 +-
.../org/apache/hadoop/hbase/io/ByteBufferPool.java | 155 -----
.../hadoop/hbase/io/compress/Compression.java | 51 +-
.../hbase/io/encoding/CopyKeyDataBlockEncoder.java | 2 +-
.../hbase/io/encoding/DiffKeyDeltaEncoder.java | 2 +-
.../hbase/io/encoding/FastDiffDeltaEncoder.java | 2 +-
.../encoding/HFileBlockDefaultDecodingContext.java | 11 +-
.../hbase/io/encoding/PrefixKeyDeltaEncoder.java | 2 +-
.../hadoop/hbase/io/encoding/RowIndexSeekerV1.java | 2 +-
.../apache/hadoop/hbase/io/hfile/HFileContext.java | 17 +-
.../apache/hadoop/hbase/io/util/BlockIOUtils.java | 254 +++++++
.../java/org/apache/hadoop/hbase/nio/ByteBuff.java | 172 +++--
.../hadoop/hbase/nio/HBaseReferenceCounted.java | 39 +-
.../org/apache/hadoop/hbase/nio/MultiByteBuff.java | 129 +++-
.../java/org/apache/hadoop/hbase/nio/RefCnt.java | 63 ++
.../apache/hadoop/hbase/nio/SingleByteBuff.java | 92 ++-
.../apache/hadoop/hbase/util/ByteBufferArray.java | 404 +++++------
.../apache/hadoop/hbase/util/ByteBufferUtils.java | 56 +-
.../hadoop/hbase/io/TestByteBuffAllocator.java | 370 ++++++++++
.../hbase/io/TestByteBufferListOutputStream.java | 18 +-
.../apache/hadoop/hbase/io/TestByteBufferPool.java | 67 --
.../apache/hadoop/hbase/nio/TestMultiByteBuff.java | 29 +-
.../hadoop/hbase/util/TestByteBufferArray.java | 185 +++--
.../hadoop/hbase/io/hfile/MemcachedBlockCache.java | 7 +-
.../regionserver/MetricsRegionServerSource.java | 15 +
.../regionserver/MetricsRegionServerWrapper.java | 28 +-
.../MetricsRegionServerSourceImpl.java | 17 +-
.../src/main/protobuf/BucketCacheEntry.proto | 79 +++
.../hbase/tmpl/regionserver/RSStatusTmpl.jamon | 3 +-
.../tmpl/regionserver/ServerMetricsTmpl.jamon | 30 +
.../apache/hadoop/hbase/io/hfile/BlockCache.java | 13 -
.../hadoop/hbase/io/hfile/BlockCacheUtil.java | 8 +-
.../apache/hadoop/hbase/io/hfile/CacheConfig.java | 21 +-
.../apache/hadoop/hbase/io/hfile/Cacheable.java | 31 +-
.../hbase/io/hfile/CacheableDeserializer.java | 31 +-
.../io/hfile/CacheableDeserializerIdManager.java | 34 +-
.../apache/hadoop/hbase/io/hfile/ChecksumUtil.java | 152 +++--
.../hadoop/hbase/io/hfile/CombinedBlockCache.java | 11 +-
.../hadoop/hbase/io/hfile/CompoundBloomFilter.java | 10 +-
.../hbase/io/hfile/ExclusiveMemHFileBlock.java | 69 ++
.../org/apache/hadoop/hbase/io/hfile/HFile.java | 6 -
.../apache/hadoop/hbase/io/hfile/HFileBlock.java | 746 ++++++++++-----------
.../hadoop/hbase/io/hfile/HFileBlockBuilder.java | 114 ++++
.../hadoop/hbase/io/hfile/HFileBlockIndex.java | 41 +-
.../hadoop/hbase/io/hfile/HFileReaderImpl.java | 304 +++++----
.../hadoop/hbase/io/hfile/LruBlockCache.java | 88 ++-
.../hadoop/hbase/io/hfile/SharedMemHFileBlock.java | 46 ++
.../hadoop/hbase/io/hfile/TinyLfuBlockCache.java | 15 +-
.../hbase/io/hfile/bucket/BucketAllocator.java | 1 -
.../hadoop/hbase/io/hfile/bucket/BucketCache.java | 694 +++++++++----------
.../hadoop/hbase/io/hfile/bucket/BucketEntry.java | 242 +++++++
.../hbase/io/hfile/bucket/BucketProtoUtils.java | 191 ++++++
.../hbase/io/hfile/bucket/ByteBufferIOEngine.java | 114 ++--
.../hbase/io/hfile/bucket/CachedEntryQueue.java | 23 +-
.../hfile/bucket/ExclusiveMemoryMmapIOEngine.java | 21 +-
.../hadoop/hbase/io/hfile/bucket/FileIOEngine.java | 42 +-
.../hbase/io/hfile/bucket/FileMmapIOEngine.java | 13 +-
.../hadoop/hbase/io/hfile/bucket/IOEngine.java | 14 +-
.../io/hfile/bucket/SharedMemoryMmapIOEngine.java | 16 +-
.../hbase/io/hfile/bucket/UniqueIndexMap.java | 56 --
.../bucket/UnsafeSharedMemoryBucketEntry.java | 81 ---
.../hadoop/hbase/ipc/NettyRpcFrameDecoder.java | 2 +-
.../apache/hadoop/hbase/ipc/NettyRpcServer.java | 2 +-
.../apache/hadoop/hbase/ipc/NettyServerCall.java | 12 +-
.../hadoop/hbase/ipc/NettyServerRpcConnection.java | 9 +-
.../org/apache/hadoop/hbase/ipc/RpcServer.java | 101 +--
.../hadoop/hbase/ipc/RpcServerInterface.java | 7 +
.../org/apache/hadoop/hbase/ipc/ServerCall.java | 18 +-
.../apache/hadoop/hbase/ipc/SimpleRpcServer.java | 2 +-
.../apache/hadoop/hbase/ipc/SimpleServerCall.java | 15 +-
.../hbase/ipc/SimpleServerRpcConnection.java | 26 +-
.../hadoop/hbase/master/MasterRpcServices.java | 5 +-
.../hadoop/hbase/mob/DefaultMobStoreCompactor.java | 28 +-
.../java/org/apache/hadoop/hbase/mob/MobCell.java | 74 ++
.../java/org/apache/hadoop/hbase/mob/MobFile.java | 21 +-
.../hadoop/hbase/regionserver/HMobStore.java | 60 +-
.../apache/hadoop/hbase/regionserver/HStore.java | 3 +-
.../MetricsRegionServerWrapperImpl.java | 33 +-
.../hadoop/hbase/regionserver/MobStoreScanner.java | 41 +-
.../hadoop/hbase/regionserver/RSRpcServices.java | 5 +-
.../regionserver/RegionServicesForStores.java | 18 +
.../regionserver/ReversedMobStoreScanner.java | 47 +-
.../hadoop/hbase/regionserver/StoreFileReader.java | 6 +-
.../hbase/regionserver/StoreFileScanner.java | 12 -
.../client/TestAsyncTableGetMultiThreaded.java | 4 +-
.../hbase/client/TestBlockEvictionFromClient.java | 51 +-
.../hbase/client/TestServerLoadDurability.java | 8 +-
.../org/apache/hadoop/hbase/io/TestHeapSize.java | 26 +
.../io/encoding/TestLoadAndSwitchEncodeOnDisk.java | 5 +-
.../hadoop/hbase/io/hfile/CacheTestUtils.java | 122 +---
...ckPositionalRead.java => TestBlockIOUtils.java} | 123 +++-
.../hadoop/hbase/io/hfile/TestCacheConfig.java | 22 +-
.../hadoop/hbase/io/hfile/TestCacheOnWrite.java | 34 +-
.../hbase/io/hfile/TestCachedBlockQueue.java | 5 -
.../apache/hadoop/hbase/io/hfile/TestChecksum.java | 116 ++--
.../hbase/io/hfile/TestCombinedBlockCache.java | 17 +
.../apache/hadoop/hbase/io/hfile/TestHFile.java | 146 +++-
.../hadoop/hbase/io/hfile/TestHFileBlock.java | 197 ++++--
.../hadoop/hbase/io/hfile/TestHFileBlockIndex.java | 11 +-
.../hbase/io/hfile/TestHFileDataBlockEncoder.java | 18 +-
.../hadoop/hbase/io/hfile/TestHFileEncryption.java | 6 +-
.../hfile/TestHFileScannerImplReferenceCount.java | 450 +++++++++++++
.../hadoop/hbase/io/hfile/TestHFileWriterV3.java | 11 +-
.../io/hfile/TestLazyDataBlockDecompression.java | 2 +-
.../hadoop/hbase/io/hfile/TestLruBlockCache.java | 91 ++-
.../apache/hadoop/hbase/io/hfile/TestPrefetch.java | 14 +-
.../hbase/io/hfile/TestTinyLfuBlockCache.java | 5 -
.../hbase/io/hfile/bucket/TestBucketCache.java | 156 +++--
.../io/hfile/bucket/TestBucketCacheRefCnt.java | 277 ++++++++
.../io/hfile/bucket/TestBucketWriterThread.java | 14 +-
.../io/hfile/bucket/TestByteBufferIOEngine.java | 122 ++--
.../bucket/TestExclusiveMemoryMmapEngine.java | 37 +-
.../hbase/io/hfile/bucket/TestFileIOEngine.java | 40 +-
.../hadoop/hbase/io/hfile/bucket/TestRAMCache.java | 127 ++++
.../org/apache/hadoop/hbase/ipc/TestRpcServer.java | 144 ----
.../hadoop/hbase/master/AbstractTestDLS.java | 29 +-
.../apache/hadoop/hbase/mob/TestCachedMobFile.java | 19 +-
.../org/apache/hadoop/hbase/mob/TestMobFile.java | 26 +-
.../hbase/mob/TestMobWithByteBuffAllocator.java | 116 ++++
.../MetricsRegionServerWrapperStub.java | 25 +
.../hadoop/hbase/regionserver/TestHMobStore.java | 19 +-
.../hbase/regionserver/TestHeapMemoryManager.java | 4 -
.../hbase/regionserver/TestRSStatusServlet.java | 29 +-
.../regionserver/TestSecureBulkLoadManager.java | 3 +-
src/main/asciidoc/_chapters/architecture.adoc | 48 --
.../asciidoc/_chapters/offheap_read_write.adoc | 186 +++++
src/main/asciidoc/book.adoc | 1 +
.../resources/images/bytebuff-allocator-stats.png | Bin 0 -> 64120 bytes
src/site/resources/images/offheap-overview.png | Bin 0 -> 64687 bytes
132 files changed, 6321 insertions(+), 3127 deletions(-)
create mode 100644
hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBuffAllocator.java
delete mode 100644
hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferPool.java
create mode 100644
hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/BlockIOUtils.java
copy
hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/CounterImpl.java
=>
hbase-common/src/main/java/org/apache/hadoop/hbase/nio/HBaseReferenceCounted.java
(58%)
create mode 100644
hbase-common/src/main/java/org/apache/hadoop/hbase/nio/RefCnt.java
create mode 100644
hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestByteBuffAllocator.java
delete mode 100644
hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestByteBufferPool.java
create mode 100644
hbase-protocol-shaded/src/main/protobuf/BucketCacheEntry.proto
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/ExclusiveMemHFileBlock.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockBuilder.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/SharedMemHFileBlock.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketEntry.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java
delete mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/UniqueIndexMap.java
delete mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/UnsafeSharedMemoryBucketEntry.java
create mode 100644
hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobCell.java
rename
hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/{TestHFileBlockPositionalRead.java
=> TestBlockIOUtils.java} (53%)
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileScannerImplReferenceCount.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCacheRefCnt.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestRAMCache.java
delete mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcServer.java
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobWithByteBuffAllocator.java
create mode 100644 src/main/asciidoc/_chapters/offheap_read_write.adoc
create mode 100644 src/site/resources/images/bytebuff-allocator-stats.png
create mode 100644 src/site/resources/images/offheap-overview.png
