minor, only apply adjustSqlDigest to involved aggregations
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d85f6ab0 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d85f6ab0 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d85f6ab0 Branch: refs/heads/master Commit: d85f6ab0130e3c1962c9dc770e06f178d3d85c32 Parents: f26b1ab Author: Roger Shi <rogershijich...@hotmail.com> Authored: Tue Aug 22 17:42:37 2017 +0800 Committer: Roger Shi <rogershijich...@gmail.com> Committed: Tue Aug 22 17:55:53 2017 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/cube/CubeCapabilityChecker.java | 5 +++++ .../extendedcolumn/ExtendedColumnMeasureType.java | 10 +++++++++- .../org/apache/kylin/measure/raw/RawMeasureType.java | 10 +++++++++- .../apache/kylin/measure/topn/TopNMeasureType.java | 15 ++++++++++++++- .../kylin/metadata/realization/CapabilityResult.java | 8 ++++++++ .../apache/kylin/metadata/realization/SQLDigest.java | 10 ++++++++-- .../apache/kylin/storage/hbase/ITStorageTest.java | 4 +++- .../kylin/query/enumerator/OLAPEnumerator.java | 1 - .../org/apache/kylin/query/relnode/OLAPContext.java | 5 ++++- .../org/apache/kylin/query/routing/QueryRouter.java | 6 ++++++ 10 files changed, 66 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/d85f6ab0/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java index 61516d2..47f268a 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java @@ -127,6 +127,11 @@ public class CubeCapabilityChecker { public double suggestCostMultiplier() { return 100; } + + @Override + public MeasureDesc getInvolvedMeasure() { + return null; + } }); } http://git-wip-us.apache.org/repos/asf/kylin/blob/d85f6ab0/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java index de5ee25..c95cd1c 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java @@ -95,6 +95,9 @@ public class ExtendedColumnMeasureType extends MeasureType<ByteArray> { @Override public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) { for (MeasureDesc measureDesc : measureDescs) { + if (!sqlDigest.involvedMeasure.contains(measureDesc)) { + continue; + } FunctionDesc extendColumnFunc = measureDesc.getFunction(); List<TblColRef> hosts = getExtendedColumnHosts(extendColumnFunc); TblColRef extended = getExtendedColumn(extendColumnFunc); @@ -111,7 +114,7 @@ public class ExtendedColumnMeasureType extends MeasureType<ByteArray> { } @Override - public CapabilityResult.CapabilityInfluence influenceCapabilityCheck(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, MeasureDesc measureDesc) { + public CapabilityResult.CapabilityInfluence influenceCapabilityCheck(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, final MeasureDesc measureDesc) { TblColRef extendedCol = getExtendedColumn(measureDesc.getFunction()); if (!unmatchedDimensions.contains(extendedCol)) { @@ -129,6 +132,11 @@ public class ExtendedColumnMeasureType extends MeasureType<ByteArray> { public double suggestCostMultiplier() { return 0.9; } + + @Override + public MeasureDesc getInvolvedMeasure() { + return measureDesc; + } }; } http://git-wip-us.apache.org/repos/asf/kylin/blob/d85f6ab0/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java index 31f35e9..eee8fa6 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java @@ -167,7 +167,7 @@ public class RawMeasureType extends MeasureType<List<ByteArray>> { return Collections.singletonList(literalCol); } - public CapabilityResult.CapabilityInfluence influenceCapabilityCheck(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, MeasureDesc measureDesc) { + public CapabilityResult.CapabilityInfluence influenceCapabilityCheck(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, final MeasureDesc measureDesc) { //is raw query if (!digest.isRawQuery) return null; @@ -185,6 +185,11 @@ public class RawMeasureType extends MeasureType<List<ByteArray>> { public double suggestCostMultiplier() { return 0.9; } + + @Override + public MeasureDesc getInvolvedMeasure() { + return measureDesc; + } }; } @@ -198,6 +203,9 @@ public class RawMeasureType extends MeasureType<List<ByteArray>> { if (sqlDigest.isRawQuery) { for (MeasureDesc measureDesc : measureDescs) { + if (!sqlDigest.involvedMeasure.contains(measureDesc)) { + continue; + } TblColRef col = this.getRawColumn(measureDesc.getFunction()); ParameterDesc colParameter = ParameterDesc.newInstance(col); FunctionDesc rawFunc = FunctionDesc.newInstance("RAW", colParameter, null); http://git-wip-us.apache.org/repos/asf/kylin/blob/d85f6ab0/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java index 89ed8ae..c33ca0f 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java @@ -233,7 +233,7 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> { } @Override - public CapabilityInfluence influenceCapabilityCheck(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, MeasureDesc topN) { + public CapabilityInfluence influenceCapabilityCheck(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, final MeasureDesc topN) { // TopN measure can (and only can) provide one numeric measure and one literal dimension // e.g. select seller, sum(gmv) from ... group by seller order by 2 desc limit 100 @@ -263,6 +263,11 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> { public double suggestCostMultiplier() { return 0.3; // make sure TopN get ahead of other matched realizations } + + @Override + public MeasureDesc getInvolvedMeasure() { + return topN; + } }; } @@ -274,6 +279,11 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> { public double suggestCostMultiplier() { return 2.0; // topn can answer but with a higher cost } + + @Override + public MeasureDesc getInvolvedMeasure() { + return topN; + } }; } @@ -318,6 +328,9 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> { } for (MeasureDesc measureDesc : measureDescs) { + if (!sqlDigest.involvedMeasure.contains(measureDesc)) { + continue; + } FunctionDesc topnFunc = measureDesc.getFunction(); List<TblColRef> topnLiteralCol = getTopNLiteralColumn(topnFunc); http://git-wip-us.apache.org/repos/asf/kylin/blob/d85f6ab0/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java index 3699b09..a2bece8 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java @@ -23,6 +23,7 @@ import java.util.List; import org.apache.kylin.metadata.model.FunctionDesc; import com.google.common.collect.Lists; +import org.apache.kylin.metadata.model.MeasureDesc; public class CapabilityResult { @@ -41,6 +42,8 @@ public class CapabilityResult { public static interface CapabilityInfluence { /** Suggest a multiplier to influence query cost */ double suggestCostMultiplier(); + + MeasureDesc getInvolvedMeasure(); } public static class DimensionAsMeasure implements CapabilityInfluence { @@ -56,6 +59,11 @@ public class CapabilityResult { return 1; } + @Override + public MeasureDesc getInvolvedMeasure() { + return null; + } + public FunctionDesc getMeasureFunction() { return function; } http://git-wip-us.apache.org/repos/asf/kylin/blob/d85f6ab0/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java index 9ce65bb..dbe517e 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java @@ -24,6 +24,7 @@ import java.util.Set; import org.apache.kylin.metadata.filter.TupleFilter; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.JoinDesc; +import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.TblColRef; import com.google.common.collect.ImmutableList; @@ -71,11 +72,14 @@ public class SQLDigest { public boolean isRawQuery; public boolean limitPrecedesAggr; + public Set<MeasureDesc> involvedMeasure; + public SQLDigest(String factTable, Set<TblColRef> allColumns, List<JoinDesc> joinDescs, // model List<TblColRef> groupbyColumns, Set<TblColRef> subqueryJoinParticipants, // group by Set<TblColRef> metricColumns, List<FunctionDesc> aggregations, List<SQLCall> aggrSqlCalls, // aggregation Set<TblColRef> filterColumns, TupleFilter filter, TupleFilter havingFilter, // filter - List<TblColRef> sortColumns, List<OrderEnum> sortOrders, boolean limitPrecedesAggr // sort & limit + List<TblColRef> sortColumns, List<OrderEnum> sortOrders, boolean limitPrecedesAggr, // sort & limit + Set<MeasureDesc> involvedMeasure ) { this.factTable = factTable; this.allColumns = allColumns; @@ -96,7 +100,9 @@ public class SQLDigest { this.sortOrders = sortOrders; this.isRawQuery = isRawQuery(); this.limitPrecedesAggr = limitPrecedesAggr; - + + this.involvedMeasure = involvedMeasure; + this.includeSubqueryJoinParticipants(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/d85f6ab0/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java index 24589a8..c61753c 100644 --- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertTrue; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import org.apache.kylin.common.KylinConfig; @@ -30,6 +31,7 @@ import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.metadata.filter.TupleFilter; import org.apache.kylin.metadata.model.FunctionDesc; +import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.realization.SQLDigest; import org.apache.kylin.metadata.realization.SQLDigest.SQLCall; @@ -139,7 +141,7 @@ public class ITStorageTest extends HBaseMetadataTestCase { groups, /*subqueryJoinParticipants*/ Sets.<TblColRef> newHashSet(), // /*metricCol*/ Collections.<TblColRef> emptySet(), aggregations, /*aggrSqlCalls*/ Collections.<SQLCall> emptyList(), // /*filter col*/ Collections.<TblColRef> emptySet(), filter, null, // - /*sortCol*/ new ArrayList<TblColRef>(), new ArrayList<SQLDigest.OrderEnum>(), false); + /*sortCol*/ new ArrayList<TblColRef>(), new ArrayList<SQLDigest.OrderEnum>(), false, new HashSet<MeasureDesc>()); iterator = storageEngine.search(context, sqlDigest, mockup.newTupleInfo(groups, aggregations)); while (iterator.hasNext()) { ITuple tuple = iterator.next(); http://git-wip-us.apache.org/repos/asf/kylin/blob/d85f6ab0/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java index 56b82b9..0af739e 100644 --- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java +++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java @@ -111,7 +111,6 @@ public class OLAPEnumerator implements Enumerator<Object[]> { // bind dynamic variables bindVariable(olapContext.filter); - // cube don't have correct result for simple query without group by, but let's try to return something makes sense olapContext.resetSQLDigest(); SQLDigest sqlDigest = olapContext.getSQLDigest(); http://git-wip-us.apache.org/repos/asf/kylin/blob/d85f6ab0/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java index 31ed075..3a42ddb 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java @@ -34,6 +34,7 @@ import org.apache.kylin.metadata.filter.TupleFilter; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.JoinDesc; import org.apache.kylin.metadata.model.JoinsTree; +import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.realization.IRealization; import org.apache.kylin.metadata.realization.SQLDigest; @@ -110,6 +111,7 @@ public class OLAPContext { public OLAPSchema olapSchema = null; public OLAPTableScan firstTableScan = null; // to be fact table scan except "select * from lookupTable" public Set<OLAPTableScan> allTableScans = new HashSet<>(); + public Set<MeasureDesc> involvedMeasure = new HashSet<>(); public TupleInfo returnTupleInfo = null; public boolean afterAggregate = false; public boolean afterHavingClauseFilter = false; @@ -156,7 +158,8 @@ public class OLAPContext { groupByColumns, subqueryJoinParticipants, // group by metricsColumns, aggregations, aggrSqlCalls, // aggregation filterColumns, filter, havingFilter, // filter - sortColumns, sortOrders, limitPrecedesAggr // sort & limit + sortColumns, sortOrders, limitPrecedesAggr, // sort & limit + involvedMeasure ); return sqlDigest; } http://git-wip-us.apache.org/repos/asf/kylin/blob/d85f6ab0/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java b/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java index 18db4ac..dbf69bc 100644 --- a/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java +++ b/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java @@ -67,6 +67,12 @@ public class QueryRouter { logger.info("The realizations remaining: " + RoutingRule.getPrintableText(candidates) + " And the final chosen one is the first one"); + for (CapabilityInfluence influence : chosen.getCapability().influences) { + if (influence.getInvolvedMeasure() != null) { + olapContext.involvedMeasure.add(influence.getInvolvedMeasure()); + } + } + return chosen.realization; }
