KYLIN-1205 Patch HBaseResourceStore as well
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7c98901b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7c98901b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7c98901b Branch: refs/heads/1.x-HBase1.1.3 Commit: 7c98901bf8acaec29504e87e210fdf5e94a4b271 Parents: 95ef74f Author: Yang Li <liy...@apache.org> Authored: Fri Dec 25 21:08:21 2015 +0800 Committer: Yang Li <liy...@apache.org> Committed: Fri Dec 25 21:08:21 2015 +0800 ---------------------------------------------------------------------- .../kylin/common/persistence/HBaseResourceStore.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/7c98901b/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java b/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java index ac14e7b..d1ff27a 100644 --- a/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java +++ b/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java @@ -148,6 +148,7 @@ public class HBaseResourceStore extends ResourceStore { Scan scan = new Scan(startRow, endRow); scan.addColumn(B_FAMILY, B_COLUMN_TS); scan.addColumn(B_FAMILY, B_COLUMN); + tuneScanParameters(scan); HTableInterface table = getConnection().getTable(getAllInOneTableName()); List<RawResource> result = Lists.newArrayList(); @@ -167,6 +168,13 @@ public class HBaseResourceStore extends ResourceStore { return result; } + private void tuneScanParameters(Scan scan) { + // divide by 10 as some resource like dictionary or snapshot can be very large + scan.setCaching(kylinConfig.getHBaseScanCacheRows() / 10); + scan.setMaxResultSize(kylinConfig.getHBaseScanMaxResultSize()); + scan.setCacheBlocks(true); + } + private InputStream getInputStream(String resPath, Result r) throws IOException { if (r == null) { return null; @@ -204,7 +212,7 @@ public class HBaseResourceStore extends ResourceStore { protected long getResourceTimestampImpl(String resPath) throws IOException { return getTimestamp(getByScan(resPath, false, true)); } - + @Override protected void putResourceImpl(String resPath, InputStream content, long ts) throws IOException { ByteArrayOutputStream bout = new ByteArrayOutputStream();
