Author: nspiegelberg
Date: Tue Oct 11 02:04:21 2011
New Revision: 1181387
URL: http://svn.apache.org/viewvc?rev=1181387&view=rev
Log:
apply HBASE 3040: BlockIndex readIndex too slowly in heavy write scenario
Summary:
Applied patches from HBASE 3040.
Test Plan:
going to fire unit tests
DiffCamp Revision: 163501
Reviewed By: kannan
CC: kannan, subbu
Revert Plan:
OK
Modified:
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
Modified:
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
URL:
http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java?rev=1181387&r1=1181386&r2=1181387&view=diff
==============================================================================
---
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
(original)
+++
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
Tue Oct 11 02:04:21 2011
@@ -23,6 +23,7 @@ import java.io.BufferedInputStream;
import java.io.Closeable;
import java.io.DataInputStream;
import java.io.DataOutputStream;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -832,6 +833,16 @@ public class HFile {
return this.inMemory;
}
+ private byte[] readAllIndex(final FSDataInputStream in, final long
indexOffset,
+ final int indexSize) throws IOException
+ {
+ byte[] allIndex = new byte[indexSize];
+ in.seek(indexOffset);
+ IOUtils.readFully(in, allIndex, 0, allIndex.length);
+
+ return allIndex;
+ }
+
/**
* Read in the index and file info.
* @return A map of fileinfo data.
@@ -852,16 +863,28 @@ public class HFile {
String clazzName = Bytes.toString(fi.get(FileInfo.COMPARATOR));
this.comparator = getComparator(clazzName);
+ int allIndexSize = (int)(this.fileSize - this.trailer.dataIndexOffset
- FixedFileTrailer.trailerSize());
+ byte[] dataAndMetaIndex = readAllIndex(this.istream,
this.trailer.dataIndexOffset, allIndexSize);
+
+ ByteArrayInputStream bis = new ByteArrayInputStream(dataAndMetaIndex);
+ DataInputStream dis = new DataInputStream(bis);
+
// Read in the data index.
- this.blockIndex = BlockIndex.readIndex(this.comparator, this.istream,
- this.trailer.dataIndexOffset, this.trailer.dataIndexCount);
+ this.blockIndex =
+ BlockIndex.readIndex(this.comparator, dis, this.trailer.dataIndexCount);
// Read in the metadata index.
if (trailer.metaIndexCount > 0) {
- this.metaIndex = BlockIndex.readIndex(Bytes.BYTES_RAWCOMPARATOR,
- this.istream, this.trailer.metaIndexOffset, trailer.metaIndexCount);
+ this.metaIndex =
BlockIndex.readIndex(Bytes.BYTES_RAWCOMPARATOR,
+ dis, this.trailer.metaIndexCount);
}
this.fileInfoLoaded = true;
+
+ if (null != dis)
+ {
+ dis.close();
+ }
+
return fi;
}
@@ -1684,12 +1707,13 @@ public class HFile {
/*
* Read in the index that is at <code>indexOffset</code>
* Must match what was written by writeIndex in the Writer.close.
+ * @param c Comparator to use.
* @param in
- * @param indexOffset
+ * @param indexSize
* @throws IOException
*/
static BlockIndex readIndex(final RawComparator<byte []> c,
- final FSDataInputStream in, final long indexOffset, final int
indexSize)
+ DataInputStream in, final int indexSize)
throws IOException {
BlockIndex bi = new BlockIndex(c);
bi.blockOffsets = new long[indexSize];
@@ -1697,9 +1721,8 @@ public class HFile {
bi.blockDataSizes = new int[indexSize];
// If index size is zero, no index was written.
if (indexSize > 0) {
- in.seek(indexOffset);
byte [] magic = new byte[INDEXBLOCKMAGIC.length];
- IOUtils.readFully(in, magic, 0, magic.length);
+ in.readFully(magic);
if (!Arrays.equals(magic, INDEXBLOCKMAGIC)) {
throw new IOException("Index block magic is wrong: " +
Arrays.toString(magic));
