KYLIN-1190 Make memory budget per query configurable Signed-off-by: honma <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a5875d03 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a5875d03 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a5875d03 Branch: refs/heads/2.0-rc Commit: a5875d033a8d9a913f5c76e44c44ebd3c7857150 Parents: 2fb097b Author: lidongsjtu <[email protected]> Authored: Thu Dec 3 09:50:19 2015 +0800 Committer: honma <[email protected]> Committed: Fri Dec 4 16:14:39 2015 +0800 ---------------------------------------------------------------------- .../main/java/org/apache/kylin/common/KylinConfig.java | 4 ++++ .../kylin/storage/hbase/cube/v1/CubeStorageQuery.java | 12 ++++++++---- .../kylin/storage/hbase/cube/v2/CubeStorageQuery.java | 11 +++++++---- 3 files changed, 19 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/a5875d03/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java index 9e6c64b..86b6a9f 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java @@ -495,6 +495,10 @@ public class KylinConfig implements Serializable { return Long.parseLong(this.getOptional("kylin.query.cache.threshold.scancount", String.valueOf(10 * 1024))); } + public long getQueryMemBudget() { + return Long.parseLong(this.getOptional("kylin.query.mem.budget", String.valueOf(3L * 1024 * 1024 * 1024))); + } + public boolean isQuerySecureEnabled() { return Boolean.parseBoolean(this.getOptional("kylin.query.security.enabled", "false")); } http://git-wip-us.apache.org/repos/asf/kylin/blob/a5875d03/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java index 4d34943..2fa0490 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java @@ -82,7 +82,6 @@ public class CubeStorageQuery implements ICachableStorageQuery { private static final Logger logger = LoggerFactory.getLogger(CubeStorageQuery.class); private static final int MERGE_KEYRANGE_THRESHOLD = 100; - private static final long MEM_BUDGET_PER_QUERY = 3L * 1024 * 1024 * 1024; // 3G private final CubeInstance cubeInstance; private final CubeDesc cubeDesc; @@ -732,8 +731,13 @@ public class CubeStorageQuery implements ICachableStorageQuery { } } - long rowEst = MEM_BUDGET_PER_QUERY / rowSizeEst; - context.setThreshold((int) rowEst); + long rowEst = this.cubeInstance.getConfig().getQueryMemBudget() / rowSizeEst; + if (rowEst > 0) { + logger.info("Memory budget is set to: " + rowEst); + context.setThreshold((int) rowEst); + } else { + logger.info("Memory budget is not set."); + } } private void setLimit(TupleFilter filter, StorageContext context) { @@ -760,7 +764,7 @@ public class CubeStorageQuery implements ICachableStorageQuery { topnLiteralCol = func.getTopNLiteralColumn(); } } - + // if TopN is not involved if (topnFunc == null) return; http://git-wip-us.apache.org/repos/asf/kylin/blob/a5875d03/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java index 0c8c3bd..59ed99b 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java @@ -41,8 +41,6 @@ public class CubeStorageQuery implements ICachableStorageQuery { private static final Logger logger = LoggerFactory.getLogger(CubeStorageQuery.class); - private static final long MEM_BUDGET_PER_QUERY = 3L * 1024 * 1024 * 1024; // 3G - private final CubeInstance cubeInstance; private final CubeDesc cubeDesc; @@ -362,8 +360,13 @@ public class CubeStorageQuery implements ICachableStorageQuery { rowSizeEst += func.getReturnDataType().getStorageBytesEstimate(); } - long rowEst = MEM_BUDGET_PER_QUERY / rowSizeEst; - context.setThreshold((int) rowEst); + long rowEst = this.cubeInstance.getConfig().getQueryMemBudget() / rowSizeEst; + if (rowEst > 0) { + logger.info("Memory budget is set to: " + rowEst); + context.setThreshold((int) rowEst); + } else { + logger.info("Memory budget is not set."); + } } private void setLimit(TupleFilter filter, StorageContext context) {
