hbase git commit: HBASE-17757 Unify blocksize after encoding to decrease memory fragment

2017-05-01 Thread anoopsamjohn 
Repository: hbase
Updated Branches:
  refs/heads/branch-1 961bb7325 -> 262c66f61


HBASE-17757 Unify blocksize after encoding to decrease memory fragment

Signed-off-by: anoopsamjohn 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/262c66f6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/262c66f6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/262c66f6

Branch: refs/heads/branch-1
Commit: 262c66f614d436c15b47ed4ae2ba33ed913c3cac
Parents: 961bb73
Author: Allan Yang 
Authored: Sun Apr 30 12:01:26 2017 +0800
Committer: anoopsamjohn 
Committed: Tue May 2 10:06:33 2017 +0530

--
 .../hadoop/hbase/io/hfile/HFileBlock.java   | 20 
 .../hadoop/hbase/io/hfile/HFileWriterV2.java| 24 +++-
 2 files changed, 38 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/262c66f6/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
index b8629da..3b014b9 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
@@ -844,6 +844,10 @@ public class HFileBlock implements Cacheable {
 // includes the header size also.
 private int unencodedDataSizeWritten;
 
+// Size of actual data being written. considering the block encoding. This
+// includes the header size also.
+private int encodedDataSizeWritten;
+
 /**
  * Bytes to be written to the file system, including the header. Compressed
  * if compression is turned on. It also includes the checksum data that
@@ -931,6 +935,7 @@ public class HFileBlock implements Cacheable {
 this.dataBlockEncoder.startBlockEncoding(dataBlockEncodingCtx, 
userDataStream);
   }
   this.unencodedDataSizeWritten = 0;
+  this.encodedDataSizeWritten = 0;
   return userDataStream;
 }
 
@@ -941,8 +946,10 @@ public class HFileBlock implements Cacheable {
  */
 void write(Cell cell) throws IOException{
   expectState(State.WRITING);
+  int posBeforeEncode = this.userDataStream.size();
   this.unencodedDataSizeWritten +=
   this.dataBlockEncoder.encode(cell, dataBlockEncodingCtx, 
this.userDataStream);
+  this.encodedDataSizeWritten += this.userDataStream.size() - 
posBeforeEncode;
 }
 
 /**
@@ -1179,6 +1186,19 @@ public class HFileBlock implements Cacheable {
  *
  * @return the number of bytes written
  */
+public int encodedBlockSizeWritten() {
+  if (state != State.WRITING)
+return 0;
+  return this.encodedDataSizeWritten;
+}
+
+/**
+ * Returns the number of bytes written into the current block so far, or
+ * zero if not writing the block at the moment. Note that this will return
+ * zero in the "block ready" state as well.
+ *
+ * @return the number of bytes written
+ */
 int blockSizeWritten() {
   if (state != State.WRITING) return 0;
   return this.unencodedDataSizeWritten;

http://git-wip-us.apache.org/repos/asf/hbase/blob/262c66f6/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
index 9e68dc3..394cd44 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
@@ -93,6 +93,14 @@ public class HFileWriterV2 extends AbstractHFileWriter {
   /** warn on cell with tags */
   private static boolean warnCellWithTags = true;
 
+
+  /** if this feature is enabled, preCalculate encoded data size before real 
encoding happens*/
+  public static final String UNIFIED_ENCODED_BLOCKSIZE_RATIO = 
"hbase.writer.unified.encoded.blocksize.ratio";
+
+  /** Block size limit after encoding, used to unify encoded block Cache entry 
size*/
+  private final int encodedBlockSizeLimit;
+
+
   static class WriterFactoryV2 extends HFile.WriterFactory {
 WriterFactoryV2(Configuration conf, CacheConfig cacheConf) {
   super(conf, cacheConf);
@@ -115,6 +123,8 @@ public class HFileWriterV2 extends AbstractHFileWriter {
 super(cacheConf,
 ostream == null ? createOutputStream(conf, fs, path, null) : ostream,
 path,

hbase git commit: HBASE-17757 Unify blocksize after encoding to decrease memory fragment

2017-04-27 Thread anoopsamjohn 
Repository: hbase
Updated Branches:
  refs/heads/master 4bc0eb31c -> c4cbb419a


HBASE-17757 Unify blocksize after encoding to decrease memory fragment

Signed-off-by: anoopsamjohn 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c4cbb419
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c4cbb419
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c4cbb419

Branch: refs/heads/master
Commit: c4cbb419a35043c3397476d32b8197cbacd21863
Parents: 4bc0eb3
Author: Allan Yang 
Authored: Fri Apr 28 08:48:00 2017 +0800
Committer: anoopsamjohn 
Committed: Fri Apr 28 07:25:36 2017 +0530

--
 .../hadoop/hbase/io/hfile/HFileBlock.java   | 20 
 .../hadoop/hbase/io/hfile/HFileWriterImpl.java  | 20 
 2 files changed, 36 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/c4cbb419/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
index 066a9fa..445dc86 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
@@ -871,6 +871,10 @@ public class HFileBlock implements Cacheable {
 // includes the header size also.
 private int unencodedDataSizeWritten;
 
+// Size of actual data being written. considering the block encoding. This
+// includes the header size also.
+private int encodedDataSizeWritten;
+
 /**
  * Bytes to be written to the file system, including the header. Compressed
  * if compression is turned on. It also includes the checksum data that
@@ -958,6 +962,7 @@ public class HFileBlock implements Cacheable {
 this.dataBlockEncoder.startBlockEncoding(dataBlockEncodingCtx, 
userDataStream);
   }
   this.unencodedDataSizeWritten = 0;
+  this.encodedDataSizeWritten = 0;
   return userDataStream;
 }
 
@@ -968,8 +973,10 @@ public class HFileBlock implements Cacheable {
  */
 void write(Cell cell) throws IOException{
   expectState(State.WRITING);
+  int posBeforeEncode = this.userDataStream.size();
   this.unencodedDataSizeWritten +=
   this.dataBlockEncoder.encode(cell, dataBlockEncodingCtx, 
this.userDataStream);
+  this.encodedDataSizeWritten += this.userDataStream.size() - 
posBeforeEncode;
 }
 
 /**
@@ -1204,6 +1211,19 @@ public class HFileBlock implements Cacheable {
  *
  * @return the number of bytes written
  */
+public int encodedBlockSizeWritten() {
+  if (state != State.WRITING)
+return 0;
+  return this.encodedDataSizeWritten;
+}
+
+/**
+ * Returns the number of bytes written into the current block so far, or
+ * zero if not writing the block at the moment. Note that this will return
+ * zero in the "block ready" state as well.
+ *
+ * @return the number of bytes written
+ */
 int blockSizeWritten() {
   if (state != State.WRITING) return 0;
   return this.unencodedDataSizeWritten;

http://git-wip-us.apache.org/repos/asf/hbase/blob/c4cbb419/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java
index 6a20b99..b258ce2 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java
@@ -63,6 +63,12 @@ public class HFileWriterImpl implements HFile.Writer {
 
   private static final long UNSET = -1;
 
+  /** if this feature is enabled, preCalculate encoded data size before real 
encoding happens*/
+  public static final String UNIFIED_ENCODED_BLOCKSIZE_RATIO = 
"hbase.writer.unified.encoded.blocksize.ratio";
+
+  /** Block size limit after encoding, used to unify encoded block Cache entry 
size*/
+  private final int encodedBlockSizeLimit;
+
   /** The Cell previously appended. Becomes the last cell in the file.*/
   protected Cell lastCell = null;
 
@@ -174,6 +180,8 @@ public class HFileWriterImpl implements HFile.Writer {
 
 closeOutputStream = path != null;
 this.cacheConf = cacheConf;
+float encodeBlockSizeRatio = 
conf.getFloat(UNIFIED_ENCODED_BLOCKSIZE_RATIO, 1f);
+this.encodedBlockSizeLimit =