Repository: hbase Updated Branches: refs/heads/0.98 6bf47a030 -> de149d0bc
HBASE-15171 Avoid counting duplicate kv and generating lots of small hfiles in PutSortReducer (Yu Li) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/38cd179b Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/38cd179b Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/38cd179b Branch: refs/heads/0.98 Commit: 38cd179bb540f0d38c5810a17097c5727947ca73 Parents: 6bf47a0 Author: tedyu <[email protected]> Authored: Wed Jan 27 09:49:45 2016 -0800 Committer: Andrew Purtell <[email protected]> Committed: Wed Feb 10 00:07:34 2016 -0800 ---------------------------------------------------------------------- .../java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/38cd179b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java index db9e585..dbf4e30 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java @@ -67,9 +67,11 @@ public class PutSortReducer extends for (Cell cell: cells) { KeyValue kv = KeyValueUtil.ensureKeyValue(cell); map.add(kv); - curSize += kv.heapSize(); } } + for(KeyValue kv: map){ + curSize +=kv.heapSize(); + } } context.setStatus("Read " + map.size() + " entries of " + map.getClass() + "(" + StringUtils.humanReadableInt(curSize) + ")");
