Repository: hbase Updated Branches: refs/heads/master 7f08cd0e1 -> fb4c50966
HBASE-17020 keylen in midkey() dont computed correctly Signed-off-by: Yu Li <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fb4c5096 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fb4c5096 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fb4c5096 Branch: refs/heads/master Commit: fb4c50966e4364cd3b34aeee26e026c3f09390ef Parents: 7f08cd0 Author: Yu Sun <[email protected]> Authored: Thu Nov 10 17:14:31 2016 +0800 Committer: Yu Li <[email protected]> Committed: Thu Nov 10 17:15:34 2016 +0800 ---------------------------------------------------------------------- .../hadoop/hbase/io/hfile/HFileBlockIndex.java | 3 +- .../hbase/io/hfile/RandomKeyValueUtil.java | 29 +++++++++ .../hbase/io/hfile/TestHFileBlockIndex.java | 68 ++++++++++++++++++++ 3 files changed, 99 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/fb4c5096/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java index 46e3261..88fe5e1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java @@ -426,7 +426,8 @@ public class HFileBlockIndex { ByteBuff b = midLeafBlock.getBufferWithoutHeader(); int numDataBlocks = b.getIntAfterPosition(0); int keyRelOffset = b.getIntAfterPosition(Bytes.SIZEOF_INT * (midKeyEntry + 1)); - int keyLen = b.getIntAfterPosition(Bytes.SIZEOF_INT * (midKeyEntry + 2)) - keyRelOffset; + int keyLen = b.getIntAfterPosition(Bytes.SIZEOF_INT * (midKeyEntry + 2)) - keyRelOffset + - SECONDARY_INDEX_ENTRY_OVERHEAD; int keyOffset = Bytes.SIZEOF_INT * (numDataBlocks + 2) + keyRelOffset + SECONDARY_INDEX_ENTRY_OVERHEAD; http://git-wip-us.apache.org/repos/asf/hbase/blob/fb4c5096/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/RandomKeyValueUtil.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/RandomKeyValueUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/RandomKeyValueUtil.java index 9507345..b6ff37e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/RandomKeyValueUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/RandomKeyValueUtil.java @@ -101,4 +101,33 @@ public class RandomKeyValueUtil { byte[] keyBytes = k.toString().getBytes(); return keyBytes; } + + public static byte[] randomOrderedFixedLengthKey(Random rand, int i, int suffixLength) { + StringBuilder k = new StringBuilder(); + + // The fixed-length lexicographically increasing part of the key. + for (int bitIndex = 31; bitIndex >= 0; --bitIndex) { + if ((i & (1 << bitIndex)) == 0) + k.append("a"); + else + k.append("b"); + } + + // A random suffix of the key. + for (int j = 0; j < suffixLength; ++j) + k.append(randomReadableChar(rand)); + + byte[] keyBytes = k.toString().getBytes(); + return keyBytes; + } + + public static byte[] randomFixedLengthValue(Random rand, int valueLength) { + StringBuilder v = new StringBuilder(); + for (int j = 0; j < valueLength; ++j) { + v.append((char) (32 + rand.nextInt(95))); + } + + byte[] valueBytes = v.toString().getBytes(); + return valueBytes; + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/fb4c5096/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java index 810dd9f..ce6ec82 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java @@ -49,6 +49,7 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.io.compress.Compression; +import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexChunk; import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexReader; @@ -514,6 +515,73 @@ public class TestHFileBlockIndex { } /** + * to check if looks good when midKey on a leaf index block boundary + * @throws IOException + */ + @Test + public void testMidKeyOnLeafIndexBlockBoundary() throws IOException { + Path hfilePath = new Path(TEST_UTIL.getDataTestDir(), + "hfile_for_midkey"); + int maxChunkSize = 512; + conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, maxChunkSize); + // should open hfile.block.index.cacheonwrite + conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, true); + + CacheConfig cacheConf = new CacheConfig(conf); + BlockCache blockCache = cacheConf.getBlockCache(); + // Evict all blocks that were cached-on-write by the previous invocation. + blockCache.evictBlocksByHfileName(hfilePath.getName()); + // Write the HFile + { + HFileContext meta = new HFileContextBuilder() + .withBlockSize(SMALL_BLOCK_SIZE) + .withCompression(Algorithm.NONE) + .withDataBlockEncoding(DataBlockEncoding.NONE) + .build(); + HFile.Writer writer = + HFile.getWriterFactory(conf, cacheConf) + .withPath(fs, hfilePath) + .withFileContext(meta) + .create(); + Random rand = new Random(19231737); + byte[] family = Bytes.toBytes("f"); + byte[] qualifier = Bytes.toBytes("q"); + int kvNumberToBeWritten = 16; + // the new generated hfile will contain 2 leaf-index blocks and 16 data blocks, + // midkey is just on the boundary of the first leaf-index block + for (int i = 0; i < kvNumberToBeWritten; ++i) { + byte[] row = RandomKeyValueUtil.randomOrderedFixedLengthKey(rand, i, 30); + + // Key will be interpreted by KeyValue.KEY_COMPARATOR + KeyValue kv = + new KeyValue(row, family, qualifier, EnvironmentEdgeManager.currentTime(), + RandomKeyValueUtil.randomFixedLengthValue(rand, SMALL_BLOCK_SIZE)); + writer.append(kv); + } + writer.close(); + } + + // close hfile.block.index.cacheonwrite + conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, false); + + // Read the HFile + HFile.Reader reader = HFile.createReader(fs, hfilePath, cacheConf, conf); + + boolean hasArrayIndexOutOfBoundsException = false; + try { + // get the mid-key. + reader.midkey(); + } catch (ArrayIndexOutOfBoundsException e) { + hasArrayIndexOutOfBoundsException = true; + } finally { + reader.close(); + } + + // to check if ArrayIndexOutOfBoundsException occured + assertFalse(hasArrayIndexOutOfBoundsException); + } + + /** * Testing block index through the HFile writer/reader APIs. Allows to test * setting index block size through configuration, intermediate-level index * blocks, and caching index blocks on write.
