KYLIN-1591 enabling different block size setting for small column families
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3aad1fc6 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3aad1fc6 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3aad1fc6 Branch: refs/heads/yang-m1 Commit: 3aad1fc60cc0278833d4b5e0e5f2db02788cd3ba Parents: ad09341 Author: Hongbin Ma <[email protected]> Authored: Mon Apr 18 15:40:05 2016 +0800 Committer: Hongbin Ma <[email protected]> Committed: Mon Apr 18 15:51:31 2016 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/common/KylinConfigBase.java | 10 +++++++--- .../apache/kylin/cube/model/HBaseColumnFamilyDesc.java | 13 +++++++++++++ .../java/org/apache/kylin/gridtable/GTScanRequest.java | 3 ++- .../kylin/storage/hbase/steps/CubeHTableUtil.java | 9 ++++++--- 4 files changed, 28 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/3aad1fc6/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 94d8f62..23f0fa7 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -481,7 +481,7 @@ abstract public class KylinConfigBase implements Serializable { } public int getQueryScanFuzzyKeyMax() { - return Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max","200")); + return Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max", "200")); } public int getHBaseKeyValueSize() { @@ -546,8 +546,12 @@ abstract public class KylinConfigBase implements Serializable { return getOptional("kylin.hbase.default.encoding", "FAST_DIFF"); } - public String getHbaseDefaultBlockSize() { - return getOptional("kylin.hbase.default.block.size", "4194304"); + public int getHbaseDefaultBlockSize() { + return Integer.valueOf(getOptional("kylin.hbase.default.block.size", "1048576")); + } + + public int getHbaseSmallFamilyBlockSize() { + return Integer.valueOf(getOptional("kylin.hbase.small.family.block.size", "65536")); } public boolean isHiveKeepFlatTable() { http://git-wip-us.apache.org/repos/asf/kylin/blob/3aad1fc6/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java index 5db2a9b..c5b2e19 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java @@ -20,6 +20,8 @@ package org.apache.kylin.cube.model; import java.util.Arrays; +import org.apache.kylin.metadata.model.MeasureDesc; + import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonProperty; @@ -55,4 +57,15 @@ public class HBaseColumnFamilyDesc { return "HBaseColumnFamilyDesc [name=" + name + ", columns=" + Arrays.toString(columns) + "]"; } + public boolean isMemoryHungry() { + for (HBaseColumnDesc hBaseColumnDesc : columns) { + for (MeasureDesc measureDesc : hBaseColumnDesc.getMeasures()) { + if (measureDesc.getFunction().getMeasureType().isMemoryHungry()) { + return true; + } + } + } + return false; + } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/3aad1fc6/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java index 97900c1..befa866 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java @@ -153,7 +153,8 @@ public class GTScanRequest { } if (!doAggr) {//Skip reading this section if you're not profiling! - int scanned = lookAndForget(result); + int scanned = result.getScannedRowCount(); + lookAndForget(result); return new EmptyGTScanner(scanned); } http://git-wip-us.apache.org/repos/asf/kylin/blob/3aad1fc6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java index 67c8518..8e3c134 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java @@ -91,6 +91,12 @@ public class CubeHTableUtil { String hbaseDefaultCC = kylinConfig.getHbaseDefaultCompressionCodec().toLowerCase(); + if (cfDesc.isMemoryHungry()) { + cf.setBlocksize(kylinConfig.getHbaseDefaultBlockSize()); + } else { + cf.setBlocksize(kylinConfig.getHbaseSmallFamilyBlockSize()); + } + switch (hbaseDefaultCC) { case "snappy": { logger.info("hbase will use snappy to compress data"); @@ -119,9 +125,6 @@ public class CubeHTableUtil { } } - int blockSize = Integer.valueOf(kylinConfig.getHbaseDefaultBlockSize()); - cf.setBlocksize(blockSize); - try { String encodingStr = kylinConfig.getHbaseDefaultEncoding(); DataBlockEncoding encoding = DataBlockEncoding.valueOf(encodingStr);
