Repository: hbase Updated Branches: refs/heads/master af33f9451 -> 7b95ac117
HBASE-16502 Reduce garbage in BufferedDataBlockEncoder - addendum adopts Hiroshi's suggestion (binlijin) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7b95ac11 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7b95ac11 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7b95ac11 Branch: refs/heads/master Commit: 7b95ac117da53ef72a9511e806b2304ebe8a0922 Parents: af33f94 Author: tedyu <[email protected]> Authored: Tue Aug 30 18:04:34 2016 -0700 Committer: tedyu <[email protected]> Committed: Tue Aug 30 18:04:34 2016 -0700 ---------------------------------------------------------------------- .../io/encoding/BufferedDataBlockEncoder.java | 33 +++++--------------- .../hbase/io/encoding/DiffKeyDeltaEncoder.java | 7 ++--- .../hbase/io/encoding/FastDiffDeltaEncoder.java | 7 ++--- .../encoding/TestBufferedDataBlockEncoder.java | 2 +- 4 files changed, 15 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/7b95ac11/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java index e8a1c3f..e3882cd 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java @@ -121,8 +121,8 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { protected boolean uncompressTags = true; /** We need to store a copy of the key. */ - protected byte[] keyBuffer = new byte[INITIAL_KEY_BUFFER_SIZE]; - protected byte[] tagsBuffer = null; + protected byte[] keyBuffer = HConstants.EMPTY_BYTE_ARRAY; + protected byte[] tagsBuffer = HConstants.EMPTY_BYTE_ARRAY; protected long memstoreTS; protected int nextKvOffset; @@ -132,15 +132,9 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { private final ObjectIntPair<ByteBuffer> tmpPair; private final boolean includeTags; - public SeekerState(ObjectIntPair<ByteBuffer> tmpPair, boolean includeTags, - boolean tagsCompressed) { + public SeekerState(ObjectIntPair<ByteBuffer> tmpPair, boolean includeTags) { this.tmpPair = tmpPair; this.includeTags = includeTags; - if (tagsCompressed) { - tagsBuffer = new byte[INITIAL_KEY_BUFFER_SIZE]; - } else { - tagsBuffer = HConstants.EMPTY_BYTE_ARRAY; - } } protected boolean isValid() { @@ -157,11 +151,8 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { protected void ensureSpaceForKey() { if (keyLength > keyBuffer.length) { - // rare case, but we need to handle arbitrary length of key - int newKeyBufferLength = Math.max(keyBuffer.length, 1) * 2; - while (keyLength > newKeyBufferLength) { - newKeyBufferLength *= 2; - } + int newKeyBufferLength = Integer.highestOneBit(Math.max( + INITIAL_KEY_BUFFER_SIZE, keyLength) - 1) << 1; byte[] newKeyBuffer = new byte[newKeyBufferLength]; System.arraycopy(keyBuffer, 0, newKeyBuffer, 0, keyBuffer.length); keyBuffer = newKeyBuffer; @@ -170,11 +161,8 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { protected void ensureSpaceForTags() { if (tagsLength > tagsBuffer.length) { - // rare case, but we need to handle arbitrary length of tags - int newTagsBufferLength = Math.max(tagsBuffer.length, 1) * 2; - while (tagsLength > newTagsBufferLength) { - newTagsBufferLength *= 2; - } + int newTagsBufferLength = Integer.highestOneBit(Math.max( + INITIAL_KEY_BUFFER_SIZE, tagsLength) - 1) << 1; byte[] newTagsBuffer = new byte[newTagsBufferLength]; System.arraycopy(tagsBuffer, 0, newTagsBuffer, 0, tagsBuffer.length); tagsBuffer = newTagsBuffer; @@ -730,10 +718,6 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { return this.decodingCtx.getHFileContext().isIncludesTags(); } - protected boolean tagsCompressed() { - return this.decodingCtx.getHFileContext().isCompressTags(); - } - @Override public int compareKey(CellComparator comparator, Cell key) { keyOnlyKV.setKey(current.keyBuffer, 0, current.keyLength); @@ -988,8 +972,7 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { protected STATE createSeekerState() { // This will fail for non-default seeker state if the subclass does not // override this method. - return (STATE) new SeekerState(this.tmpPair, this.includesTags(), - this.tagsCompressed()); + return (STATE) new SeekerState(this.tmpPair, this.includesTags()); } abstract protected void decodeFirst(); http://git-wip-us.apache.org/repos/asf/hbase/blob/7b95ac11/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java index 0caf8e8..9f42ad3 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java @@ -368,8 +368,8 @@ public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder { private long timestamp; public DiffSeekerState(ObjectIntPair<ByteBuffer> tmpPair, - boolean includeTags, boolean tagsCompressed) { - super(tmpPair, includeTags, tagsCompressed); + boolean includeTags) { + super(tmpPair, includeTags); } @Override @@ -504,8 +504,7 @@ public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder { @Override protected DiffSeekerState createSeekerState() { - return new DiffSeekerState(this.tmpPair, this.includesTags(), - this.tagsCompressed()); + return new DiffSeekerState(this.tmpPair, this.includesTags()); } }; } http://git-wip-us.apache.org/repos/asf/hbase/blob/7b95ac11/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java index b8b56ef..85b33f8 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java @@ -379,8 +379,8 @@ public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder { private int familyLengthWithSize; public FastDiffSeekerState(ObjectIntPair<ByteBuffer> tmpPair, - boolean includeTags, boolean tagsCompressed) { - super(tmpPair, includeTags, tagsCompressed); + boolean includeTags) { + super(tmpPair, includeTags); } @Override @@ -520,8 +520,7 @@ public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder { @Override protected FastDiffSeekerState createSeekerState() { - return new FastDiffSeekerState(this.tmpPair, this.includesTags(), - this.tagsCompressed()); + return new FastDiffSeekerState(this.tmpPair, this.includesTags()); } }; } http://git-wip-us.apache.org/repos/asf/hbase/blob/7b95ac11/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java index d31fd0c..d7e5bf5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java @@ -49,7 +49,7 @@ public class TestBufferedDataBlockEncoder { @Test public void testEnsureSpaceForKey() { BufferedDataBlockEncoder.SeekerState state = new BufferedDataBlockEncoder.SeekerState( - new ObjectIntPair<ByteBuffer>(), false, false); + new ObjectIntPair<ByteBuffer>(), false); for (int i = 1; i <= 65536; ++i) { state.keyLength = i; state.ensureSpaceForKey();
