Repository: hbase Updated Branches: refs/heads/branch-1 961bb7325 -> 262c66f61
HBASE-17757 Unify blocksize after encoding to decrease memory fragment Signed-off-by: anoopsamjohn <anoopsamj...@gmail.com> Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/262c66f6 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/262c66f6 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/262c66f6 Branch: refs/heads/branch-1 Commit: 262c66f614d436c15b47ed4ae2ba33ed913c3cac Parents: 961bb73 Author: Allan Yang <allan...@163.com> Authored: Sun Apr 30 12:01:26 2017 +0800 Committer: anoopsamjohn <anoopsamj...@gmail.com> Committed: Tue May 2 10:06:33 2017 +0530 ---------------------------------------------------------------------- .../hadoop/hbase/io/hfile/HFileBlock.java | 20 ++++++++++++++++ .../hadoop/hbase/io/hfile/HFileWriterV2.java | 24 +++++++++++++++----- 2 files changed, 38 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/262c66f6/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java index b8629da..3b014b9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java @@ -844,6 +844,10 @@ public class HFileBlock implements Cacheable { // includes the header size also. private int unencodedDataSizeWritten; + // Size of actual data being written. considering the block encoding. This + // includes the header size also. + private int encodedDataSizeWritten; + /** * Bytes to be written to the file system, including the header. Compressed * if compression is turned on. It also includes the checksum data that @@ -931,6 +935,7 @@ public class HFileBlock implements Cacheable { this.dataBlockEncoder.startBlockEncoding(dataBlockEncodingCtx, userDataStream); } this.unencodedDataSizeWritten = 0; + this.encodedDataSizeWritten = 0; return userDataStream; } @@ -941,8 +946,10 @@ public class HFileBlock implements Cacheable { */ void write(Cell cell) throws IOException{ expectState(State.WRITING); + int posBeforeEncode = this.userDataStream.size(); this.unencodedDataSizeWritten += this.dataBlockEncoder.encode(cell, dataBlockEncodingCtx, this.userDataStream); + this.encodedDataSizeWritten += this.userDataStream.size() - posBeforeEncode; } /** @@ -1179,6 +1186,19 @@ public class HFileBlock implements Cacheable { * * @return the number of bytes written */ + public int encodedBlockSizeWritten() { + if (state != State.WRITING) + return 0; + return this.encodedDataSizeWritten; + } + + /** + * Returns the number of bytes written into the current block so far, or + * zero if not writing the block at the moment. Note that this will return + * zero in the "block ready" state as well. + * + * @return the number of bytes written + */ int blockSizeWritten() { if (state != State.WRITING) return 0; return this.unencodedDataSizeWritten; http://git-wip-us.apache.org/repos/asf/hbase/blob/262c66f6/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java index 9e68dc3..394cd44 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java @@ -93,6 +93,14 @@ public class HFileWriterV2 extends AbstractHFileWriter { /** warn on cell with tags */ private static boolean warnCellWithTags = true; + + /** if this feature is enabled, preCalculate encoded data size before real encoding happens*/ + public static final String UNIFIED_ENCODED_BLOCKSIZE_RATIO = "hbase.writer.unified.encoded.blocksize.ratio"; + + /** Block size limit after encoding, used to unify encoded block Cache entry size*/ + private final int encodedBlockSizeLimit; + + static class WriterFactoryV2 extends HFile.WriterFactory { WriterFactoryV2(Configuration conf, CacheConfig cacheConf) { super(conf, cacheConf); @@ -115,6 +123,8 @@ public class HFileWriterV2 extends AbstractHFileWriter { super(cacheConf, ostream == null ? createOutputStream(conf, fs, path, null) : ostream, path, comparator, context); + float encodeBlockSizeRatio = conf.getFloat(UNIFIED_ENCODED_BLOCKSIZE_RATIO, 1f); + this.encodedBlockSizeLimit = (int)(hFileContext.getBlocksize() * encodeBlockSizeRatio); finishInit(conf); } @@ -147,12 +157,14 @@ public class HFileWriterV2 extends AbstractHFileWriter { * @throws IOException */ protected void checkBlockBoundary() throws IOException { - if (fsBlockWriter.blockSizeWritten() < hFileContext.getBlocksize()) - return; - - finishBlock(); - writeInlineBlocks(false); - newBlock(); + //for encoder like prefixTree, encoded size is not available, so we have to compare both encoded size + //and unencoded size to blocksize limit. + if (fsBlockWriter.encodedBlockSizeWritten() >= encodedBlockSizeLimit + || fsBlockWriter.blockSizeWritten() >= hFileContext.getBlocksize()) { + finishBlock(); + writeInlineBlocks(false); + newBlock(); + } } /** Clean up the current data block */