Repository: kylin Updated Branches: refs/heads/yang-m1 96eca30d8 -> 29b3914a2
KYLIN-1585 make MAX_HBASE_FUZZY_KEYS in GTScanRangePlanner configurable Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/29b3914a Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/29b3914a Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/29b3914a Branch: refs/heads/yang-m1 Commit: 29b3914a26a1d94ecfbb89ed942d067966bdf86a Parents: 96eca30 Author: Hongbin Ma <[email protected]> Authored: Thu Apr 14 12:12:13 2016 +0800 Committer: Hongbin Ma <[email protected]> Committed: Thu Apr 14 12:12:13 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 4 +++ .../kylin/gridtable/GTScanRangePlanner.java | 35 ++++++++++++++------ 2 files changed, 28 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/29b3914a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 4a5106e..94d8f62 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -480,6 +480,10 @@ abstract public class KylinConfigBase implements Serializable { return Boolean.parseBoolean(this.getOptional("kylin.query.ignore_unknown_function", "false")); } + public int getQueryScanFuzzyKeyMax() { + return Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max","200")); + } + public int getHBaseKeyValueSize() { return Integer.parseInt(this.getOptional("kylin.hbase.client.keyvalue.maxsize", "10485760")); } http://git-wip-us.apache.org/repos/asf/kylin/blob/29b3914a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java index 2307aaf..90b97ad 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java @@ -18,9 +18,19 @@ package org.apache.kylin.gridtable; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.debug.BackdoorToggles; import org.apache.kylin.common.util.ByteArray; import org.apache.kylin.common.util.ImmutableBitSet; @@ -35,13 +45,15 @@ import org.apache.kylin.metadata.model.TblColRef; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; public class GTScanRangePlanner { private static final Logger logger = LoggerFactory.getLogger(GTScanRangePlanner.class); - private static final int MAX_HBASE_FUZZY_KEYS = 100; + private int maxFuzzyKey = 200; final private GTInfo info; final private Pair<ByteArray, ByteArray> segmentStartAndEnd; @@ -70,6 +82,8 @@ public class GTScanRangePlanner { this.rangeEndComparator = getRangeEndComparator(comp); //start key GTRecord compare to stop key GTRecord this.rangeStartEndComparator = getRangeStartEndComparator(comp); + + maxFuzzyKey = KylinConfig.getInstanceFromEnv().getQueryScanFuzzyKeyMax(); } // return empty list meaning filter is always false @@ -116,11 +130,11 @@ public class GTScanRangePlanner { if (partitionColRef != null && range.column.equals(partitionColRef)) { if (rangeStartEndComparator.comparator.compare(segmentStartAndEnd.getFirst(), range.end) <= 0 // && (rangeStartEndComparator.comparator.compare(range.begin, segmentStartAndEnd.getSecond()) < 0 // - || rangeStartEndComparator.comparator.compare(range.begin, segmentStartAndEnd.getSecond()) == 0 // - && (range.op == FilterOperatorEnum.EQ || range.op == FilterOperatorEnum.LTE || range.op == FilterOperatorEnum.GTE || range.op == FilterOperatorEnum.IN))) { + || rangeStartEndComparator.comparator.compare(range.begin, segmentStartAndEnd.getSecond()) == 0 // + && (range.op == FilterOperatorEnum.EQ || range.op == FilterOperatorEnum.LTE || range.op == FilterOperatorEnum.GTE || range.op == FilterOperatorEnum.IN))) { //segment range is [Closed,Open), but segmentStartAndEnd.getSecond() might be rounded, so use <= when has equals in condition. } else { - logger.debug("Pre-check partition col filter failed, partitionColRef {}, segment start {}, segment end {}, range begin {}, range end {}",// + logger.debug("Pre-check partition col filter failed, partitionColRef {}, segment start {}, segment end {}, range begin {}, range end {}", // new Object[] { partitionColRef, makeReadable(segmentStartAndEnd.getFirst()), makeReadable(segmentStartAndEnd.getSecond()), makeReadable(range.begin), makeReadable(range.end) }); return null; } @@ -154,9 +168,8 @@ public class GTScanRangePlanner { logger.info("The execution of this query will not use fuzzy key"); return result; } - - List<Map<Integer, ByteArray>> fuzzyValueCombinations = FuzzyValueCombination.calculate(fuzzyValueSet, MAX_HBASE_FUZZY_KEYS); + List<Map<Integer, ByteArray>> fuzzyValueCombinations = FuzzyValueCombination.calculate(fuzzyValueSet, maxFuzzyKey); for (Map<Integer, ByteArray> fuzzyValue : fuzzyValueCombinations) { @@ -332,7 +345,7 @@ public class GTScanRangePlanner { // if any range is non-fuzzy, then all fuzzy keys must be cleared // also too many fuzzy keys will slow down HBase scan - if (hasNonFuzzyRange || newFuzzyKeys.size() > MAX_HBASE_FUZZY_KEYS) { + if (hasNonFuzzyRange || newFuzzyKeys.size() > maxFuzzyKey) { newFuzzyKeys.clear(); }
