minor, refine query error message
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b97e16ad Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b97e16ad Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b97e16ad Branch: refs/heads/security_update Commit: b97e16ad8ad561e2f1ca8fe60fe7ad8c577401d8 Parents: 6d47976 Author: liveandevil <1361605...@qq.com> Authored: Sat Sep 16 14:51:48 2017 +0800 Committer: Dong Li <lid...@apache.org> Committed: Sat Sep 16 15:30:30 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 8 +- .../kylin/cube/CubeCapabilityChecker.java | 54 +++-- .../org/apache/kylin/cube/CubeInstance.java | 10 - .../kylin/cube/model/AggregationGroup.java | 7 +- .../org/apache/kylin/cube/model/CubeDesc.java | 13 +- .../kylin/metadata/model/DataModelDesc.java | 6 +- .../kylin/metadata/model/JoinTableDesc.java | 50 +++- .../apache/kylin/metadata/model/JoinsTree.java | 34 ++- .../metadata/realization/CapabilityResult.java | 75 +++++- .../kylin/storage/hybrid/HybridInstance.java | 4 +- .../apache/kylin/query/relnode/OLAPContext.java | 7 +- .../apache/kylin/query/relnode/OLAPJoinRel.java | 14 +- .../apache/kylin/query/routing/QueryRouter.java | 25 +- .../kylin/query/routing/RealizationCheck.java | 242 +++++++++++++++++++ .../kylin/query/routing/RealizationChooser.java | 42 +++- .../rules/RemoveUncapableRealizationsRule.java | 5 +- .../org/apache/kylin/query/util/QueryUtil.java | 7 +- .../apache/kylin/rest/service/QueryService.java | 6 +- .../rest/controller/QueryControllerTest.java | 7 +- 19 files changed, 534 insertions(+), 82 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 990736f..ab19d2a 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -290,7 +290,7 @@ abstract public class KylinConfigBase implements Serializable { "org.apache.kylin.storage.hbase.util.ZookeeperDistributedLock$Factory"); return (DistributedLockFactory) ClassUtil.newInstance(clsName); } - + public String getHBaseMappingAdapter() { return getOptional("kylin.metadata.hbasemapping-adapter"); } @@ -346,7 +346,7 @@ abstract public class KylinConfigBase implements Serializable { public String getSegmentAdvisor() { return getOptional("kylin.cube.segment-advisor", "org.apache.kylin.cube.CubeSegmentAdvisor"); } - + public double getJobCuboidSizeRatio() { return Double.parseDouble(getOptional("kylin.cube.size-estimate-ratio", "0.25")); } @@ -388,6 +388,10 @@ abstract public class KylinConfigBase implements Serializable { return Boolean.parseBoolean(getOptional("kylin.cube.aggrgroup.is-mandatory-only-valid", "false")); } + public int getCubeRowkeyMaxSize() { + return Integer.parseInt(getOptional("kylin.cube.rowkey.max-size", "63")); + } + public int getMaxBuildingSegments() { return Integer.parseInt(getOptional("kylin.cube.max-building-segments", "10")); } http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java index 47f268a..5dffd96 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java @@ -70,7 +70,8 @@ public class CubeCapabilityChecker { //1. dimension as measure if (!unmatchedAggregations.isEmpty()) { - tryDimensionAsMeasures(unmatchedAggregations, result, cube.getDescriptor().listDimensionColumnsIncludingDerived()); + tryDimensionAsMeasures(unmatchedAggregations, result, + cube.getDescriptor().listDimensionColumnsIncludingDerived()); } } else { //for non query-on-facttable @@ -103,36 +104,47 @@ public class CubeCapabilityChecker { if (!unmatchedDimensions.isEmpty()) { logger.info("Exclude cube " + cube.getName() + " because unmatched dimensions: " + unmatchedDimensions); + result.incapableCause = CapabilityResult.IncapableCause.unmatchedDimensions(unmatchedDimensions); return result; } if (!unmatchedAggregations.isEmpty()) { logger.info("Exclude cube " + cube.getName() + " because unmatched aggregations: " + unmatchedAggregations); + result.incapableCause = CapabilityResult.IncapableCause.unmatchedAggregations(unmatchedAggregations); return result; } - if (cube.getStorageType() == IStorageAware.ID_HBASE && MassInTupleFilter.containsMassInTupleFilter(digest.filter)) { - logger.info("Exclude cube " + cube.getName() + " because only v2 storage + v2 query engine supports massin"); + if (cube.getStorageType() == IStorageAware.ID_HBASE + && MassInTupleFilter.containsMassInTupleFilter(digest.filter)) { + logger.info( + "Exclude cube " + cube.getName() + " because only v2 storage + v2 query engine supports massin"); + result.incapableCause = CapabilityResult.IncapableCause.create(CapabilityResult.IncapableType.UNSUPPORT_MASSIN); return result; } if (digest.limitPrecedesAggr) { logger.info("Exclude cube " + cube.getName() + " because there's limit preceding aggregation"); + result.incapableCause = CapabilityResult.IncapableCause.create(CapabilityResult.IncapableType.LIMIT_PRECEDE_AGGR); return result; } if (digest.isRawQuery && rootFactTable.equals(digest.factTable)) { - result.influences.add(new CapabilityInfluence() { - @Override - public double suggestCostMultiplier() { - return 100; - } + if (cube.getConfig().isDisableCubeNoAggSQL()) { + result.incapableCause = CapabilityResult.IncapableCause.create(CapabilityResult.IncapableType.UNSUPPORT_RAWQUERY); + return result; + } else { + result.influences.add(new CapabilityInfluence() { + @Override + public double suggestCostMultiplier() { + return 100; + } - @Override - public MeasureDesc getInvolvedMeasure() { - return null; - } - }); + @Override + public MeasureDesc getInvolvedMeasure() { + return null; + } + }); + } } // cost will be minded by caller @@ -164,7 +176,8 @@ public class CubeCapabilityChecker { return result; } - private static void tryDimensionAsMeasures(Collection<FunctionDesc> unmatchedAggregations, CapabilityResult result, Set<TblColRef> dimCols) { + private static void tryDimensionAsMeasures(Collection<FunctionDesc> unmatchedAggregations, CapabilityResult result, + Set<TblColRef> dimCols) { Iterator<FunctionDesc> it = unmatchedAggregations.iterator(); while (it.hasNext()) { @@ -182,7 +195,8 @@ public class CubeCapabilityChecker { continue; } List<TblColRef> neededCols = parameterDesc.getColRefs(); - if (neededCols.size() > 0 && dimCols.containsAll(neededCols) && FunctionDesc.BUILT_IN_AGGREGATIONS.contains(functionDesc.getExpression())) { + if (neededCols.size() > 0 && dimCols.containsAll(neededCols) + && FunctionDesc.BUILT_IN_AGGREGATIONS.contains(functionDesc.getExpression())) { result.influences.add(new CapabilityResult.DimensionAsMeasure(functionDesc)); it.remove(); continue; @@ -191,7 +205,9 @@ public class CubeCapabilityChecker { } // custom measure types can cover unmatched dimensions or measures - private static void tryCustomMeasureTypes(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, CubeInstance cube, CapabilityResult result) { + private static void tryCustomMeasureTypes(Collection<TblColRef> unmatchedDimensions, + Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, CubeInstance cube, + CapabilityResult result) { CubeDesc cubeDesc = cube.getDescriptor(); List<String> influencingMeasures = Lists.newArrayList(); for (MeasureDesc measure : cubeDesc.getMeasures()) { @@ -202,14 +218,16 @@ public class CubeCapabilityChecker { if (measureType instanceof BasicMeasureType) continue; - CapabilityInfluence inf = measureType.influenceCapabilityCheck(unmatchedDimensions, unmatchedAggregations, digest, measure); + CapabilityInfluence inf = measureType.influenceCapabilityCheck(unmatchedDimensions, unmatchedAggregations, + digest, measure); if (inf != null) { result.influences.add(inf); influencingMeasures.add(measure.getName() + "@" + measureType.getClass()); } } if (influencingMeasures.size() != 0) - logger.info("Cube {} CapabilityInfluences: {}", cube.getCanonicalName(), StringUtils.join(influencingMeasures, ",")); + logger.info("Cube {} CapabilityInfluences: {}", cube.getCanonicalName(), + StringUtils.join(influencingMeasures, ",")); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java index 7d539c6..ac41970 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java @@ -404,7 +404,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, @Override public CapabilityResult isCapable(SQLDigest digest) { CapabilityResult result = CubeCapabilityChecker.check(this, digest); - result = localCapacityCheck(digest, result); if (result.capable) { result.cost = getCost(digest); for (CapabilityInfluence i : result.influences) { @@ -416,15 +415,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, return result; } - private CapabilityResult localCapacityCheck(SQLDigest digest, CapabilityResult originResult) { - if (this.getDescriptor().getConfig().isDisableCubeNoAggSQL()) { - CapabilityResult notCap = new CapabilityResult(); - notCap.capable = false; - return digest.aggregations.isEmpty() ? notCap : originResult ; - } - return originResult; - } - public int getCost(SQLDigest digest) { int calculatedCost = cost; http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java index 8c90b8e..d473858 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java @@ -122,7 +122,8 @@ public class AggregationGroup implements Serializable { // check no dup Set<String> set = new HashSet<>(Arrays.asList(names)); if (set.size() < names.length) - throw new IllegalStateException("Columns in aggrgroup must not contain duplication: " + Arrays.asList(names)); + throw new IllegalStateException( + "Columns in aggrgroup must not contain duplication: " + Arrays.asList(names)); } private void buildPartialCubeFullMask(RowKeyDesc rowKeyDesc) { @@ -325,12 +326,16 @@ public class AggregationGroup implements Serializable { normalDims.removeAll(jointDims); combination = combination * (1L << normalDims.size()); + if (cubeDesc.getConfig().getCubeAggrGroupIsMandatoryOnlyValid() && !mandatoryDims.isEmpty()) { combination += 1; } combination -= 1; // not include cuboid 0 } + if (combination < 0) { // overflow + combination = Long.MAX_VALUE - 1; + } return combination; } http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 46683c4..f2dab51 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -96,8 +96,6 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { } } - public static final int MAX_ROWKEY_SIZE = 64; - public enum DeriveType implements java.io.Serializable { LOOKUP, PK_FK, EXTENDED_COLUMN } @@ -191,7 +189,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { private Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedMap = Maps.newHashMap(); private Map<TblColRef, DeriveInfo> extendedColumnToHosts = Maps.newHashMap(); - + transient private CuboidScheduler cuboidScheduler = null; public boolean isEnableSharding() { @@ -563,9 +561,6 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { checkArgument(StringUtils.isNotBlank(name), "CubeDesc name is blank"); checkArgument(StringUtils.isNotBlank(modelName), "CubeDesc (%s) has blank model name", name); - checkArgument(this.rowkey.getRowKeyColumns().length < MAX_ROWKEY_SIZE, - "Too many rowkeys (%s) in CubeDesc, please try to reduce dimension number or adopt derived dimensions", - this.rowkey.getRowKeyColumns().length); // note CubeDesc.name == CubeInstance.name List<ProjectInstance> ownerPrj = ProjectManager.getInstance(config).findProjects(RealizationType.CUBE, name); @@ -582,6 +577,10 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { this.config = KylinConfigExt.createInstance(config, overrideKylinProps); + checkArgument(this.rowkey.getRowKeyColumns().length <= this.config.getCubeRowkeyMaxSize(), + "Too many rowkeys (%s) in CubeDesc, please try to reduce dimension number or adopt derived dimensions", + this.rowkey.getRowKeyColumns().length); + this.model = MetadataManager.getInstance(config).getDataModelDesc(modelName); checkNotNull(this.model, "DateModelDesc(%s) not found", modelName); @@ -618,7 +617,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { hbaseMapping.init(this); initMeasureReferenceToColumnFamily(); } - } + } // check all dimension columns are presented on rowkey List<TblColRef> dimCols = listDimensionColumnsExcludingDerived(true); http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java index b27d644..1897c0f 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java @@ -843,12 +843,10 @@ public class DataModelDesc extends RootPersistentEntity { return metrics; } - @Deprecated public void setDimensions(List<ModelDimensionDesc> dimensions) { this.dimensions = dimensions; } - @Deprecated public void setMetrics(String[] metrics) { this.metrics = metrics; } @@ -873,9 +871,11 @@ public class DataModelDesc extends RootPersistentEntity { copy.dimensions = orig.dimensions; copy.metrics = orig.metrics; copy.filterCondition = orig.filterCondition; - copy.partitionDesc = PartitionDesc.getCopyOf(orig.getPartitionDesc()); copy.capacity = orig.capacity; copy.computedColumnDescs = orig.computedColumnDescs; + if (orig.getPartitionDesc() != null) { + copy.partitionDesc = PartitionDesc.getCopyOf(orig.getPartitionDesc()); + } copy.updateRandomUuid(); return copy; } http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java index 56c90bd..dc4710e 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java @@ -37,36 +37,44 @@ public class JoinTableDesc implements Serializable { @JsonProperty("kind") @JsonInclude(JsonInclude.Include.NON_NULL) private TableKind kind = TableKind.LOOKUP; - + @JsonProperty("alias") @JsonInclude(JsonInclude.Include.NON_NULL) private String alias; - + @JsonProperty("join") private JoinDesc join; - + private TableRef tableRef; public String getTable() { return table; } - void setTable(String table) { + public void setTable(String table) { this.table = table; } public TableKind getKind() { return kind; } - - void setAlias(String alias) { + + public void setKind(TableKind kind) { + this.kind = kind; + } + + public void setAlias(String alias) { this.alias = alias; } - + public String getAlias() { return alias; } - + + public void setJoin(JoinDesc join) { + this.join = join; + } + public JoinDesc getJoin() { return join; } @@ -79,4 +87,30 @@ public class JoinTableDesc implements Serializable { this.tableRef = ref; } + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + JoinTableDesc that = (JoinTableDesc) o; + + if (table != null ? !table.equals(that.table) : that.table != null) + return false; + if (kind != that.kind) + return false; + if (alias != null ? !alias.equals(that.alias) : that.alias != null) + return false; + return join != null ? join.equals(that.join) : that.join == null; + } + + @Override + public int hashCode() { + int result = table != null ? table.hashCode() : 0; + result = 31 * result + (kind != null ? kind.hashCode() : 0); + result = 31 * result + (alias != null ? alias.hashCode() : 0); + result = 31 * result + (join != null ? join.hashCode() : 0); + return result; + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java index 4e7e8b8..a37c6be 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; public class JoinsTree implements Serializable { private static final long serialVersionUID = 1L; @@ -73,7 +74,8 @@ public class JoinsTree implements Serializable { return matchUp.size(); } - private boolean matchInTree(Chain chain, JoinsTree another, Map<String, String> constraints, Map<String, String> matchUp) { + private boolean matchInTree(Chain chain, JoinsTree another, Map<String, String> constraints, + Map<String, String> matchUp) { String thisAlias = chain.table.getAlias(); if (matchUp.containsKey(thisAlias)) return true; @@ -103,7 +105,8 @@ public class JoinsTree implements Serializable { boolean matches = false; if (chain.join == null) { - matches = anotherChain.join == null && chain.table.getTableDesc().getIdentity().equals(anotherChain.table.getTableDesc().getIdentity()); + matches = anotherChain.join == null + && chain.table.getTableDesc().getIdentity().equals(anotherChain.table.getTableDesc().getIdentity()); } else { matches = chain.join.matches(anotherChain.join) && matchChain(chain.fkSide, anotherChain.fkSide, matchUp); } @@ -122,6 +125,21 @@ public class JoinsTree implements Serializable { return chain.join; } + public List<Chain> unmatchedChain(JoinsTree another, Map<String, String> constraints) { + Map<String, String> matchUp = new HashMap<>(); + List<Chain> unmatchedChainList = Lists.newArrayList(); + for (Chain chain : tableChains.values()) { + if (matchInTree(chain, another, constraints, matchUp) == false) + unmatchedChainList.add(chain); + } + + return unmatchedChainList; + } + + public Map<String, Chain> getTableChains() { + return tableChains; + } + public static class Chain implements Serializable { private static final long serialVersionUID = 1L; @@ -138,6 +156,18 @@ public class JoinsTree implements Serializable { Preconditions.checkArgument(fkSide.table == join.getFKSide()); } } + + public JoinDesc getJoin() { + return join; + } + + public TableRef getTable() { + return table; + } + + public Chain getFkSide() { + return fkSide; + } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java index a2bece8..ba21939 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java @@ -18,29 +18,42 @@ package org.apache.kylin.metadata.realization; +import java.util.Collection; import java.util.List; import org.apache.kylin.metadata.model.FunctionDesc; +import org.apache.kylin.metadata.model.MeasureDesc; +import org.apache.kylin.metadata.model.TblColRef; import com.google.common.collect.Lists; -import org.apache.kylin.metadata.model.MeasureDesc; public class CapabilityResult { - /** Is capable or not */ + /** + * Is capable or not + */ public boolean capable; - /** The smaller the cost, the more capable the realization */ + /** + * The smaller the cost, the more capable the realization + */ public int cost; /** + * reason of incapable + */ + public IncapableCause incapableCause; + + /** * Marker objects to indicate all special features * (dimension-as-measure, topN etc.) that have influenced the capability check. */ public List<CapabilityInfluence> influences = Lists.newArrayListWithCapacity(1); public static interface CapabilityInfluence { - /** Suggest a multiplier to influence query cost */ + /** + * Suggest a multiplier to influence query cost + */ double suggestCostMultiplier(); MeasureDesc getInvolvedMeasure(); @@ -68,4 +81,58 @@ public class CapabilityResult { return function; } } + + public static enum IncapableType { + UNMATCHED_DIMENSION, UNMATCHED_AGGREGATION, UNSUPPORT_MASSIN, UNSUPPORT_RAWQUERY, LIMIT_PRECEDE_AGGR, II_UNMATCHED_FACT_TABLE, II_MISSING_COLS + } + + public static class IncapableCause { + private IncapableType incapableType; + private Collection<TblColRef> unmatchedDimensions; + private Collection<FunctionDesc> unmatchedAggregations; + + public static IncapableCause unmatchedDimensions(Collection<TblColRef> unmatchedDimensions) { + IncapableCause incapableCause = new IncapableCause(); + incapableCause.setIncapableType(IncapableType.UNMATCHED_DIMENSION); + incapableCause.setUnmatchedDimensions(unmatchedDimensions); + return incapableCause; + } + + public static IncapableCause unmatchedAggregations(Collection<FunctionDesc> unmatchedAggregations) { + IncapableCause incapableCause = new IncapableCause(); + incapableCause.setIncapableType(IncapableType.UNMATCHED_AGGREGATION); + incapableCause.setUnmatchedAggregations(unmatchedAggregations); + return incapableCause; + } + + public static IncapableCause create(IncapableType incapableType) { + IncapableCause incapableCause = new IncapableCause(); + incapableCause.setIncapableType(incapableType); + return incapableCause; + } + + public IncapableType getIncapableType() { + return incapableType; + } + + public void setIncapableType(IncapableType incapableType) { + this.incapableType = incapableType; + } + + public Collection<TblColRef> getUnmatchedDimensions() { + return unmatchedDimensions; + } + + public void setUnmatchedDimensions(Collection<TblColRef> unmatchedDimensions) { + this.unmatchedDimensions = unmatchedDimensions; + } + + public Collection<FunctionDesc> getUnmatchedAggregations() { + return unmatchedAggregations; + } + + public void setUnmatchedAggregations(Collection<FunctionDesc> unmatchedAggregations) { + this.unmatchedAggregations = unmatchedAggregations; + } + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java index 1b113ee..df68e10 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java @@ -188,6 +188,8 @@ public class HybridInstance extends RootPersistentEntity implements IRealization result.capable = true; result.cost = Math.min(result.cost, child.cost); result.influences.addAll(child.influences); + } else { + result.incapableCause = child.incapableCause; } } @@ -220,7 +222,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization init(); return allColumnDescs; } - + @Override public List<MeasureDesc> getMeasures() { init(); http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java index d1608e9..c5c4c44 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java @@ -41,6 +41,7 @@ import org.apache.kylin.metadata.realization.IRealization; import org.apache.kylin.metadata.realization.SQLDigest; import org.apache.kylin.metadata.realization.SQLDigest.SQLCall; import org.apache.kylin.metadata.tuple.TupleInfo; +import org.apache.kylin.query.routing.RealizationCheck; import org.apache.kylin.query.schema.OLAPSchema; import org.apache.kylin.storage.StorageContext; import org.apache.kylin.storage.hybrid.HybridInstance; @@ -124,6 +125,7 @@ public class OLAPContext { // cube metadata public IRealization realization; + public RealizationCheck realizationCheck; public Set<TblColRef> allColumns = new HashSet<>(); public List<TblColRef> groupByColumns = new ArrayList<>(); @@ -161,8 +163,7 @@ public class OLAPContext { metricsColumns, aggregations, aggrSqlCalls, // aggregation filterColumns, filter, havingFilter, // filter sortColumns, sortOrders, limitPrecedesAggr, // sort & limit - involvedMeasure - ); + involvedMeasure); return sqlDigest; } @@ -180,7 +181,7 @@ public class OLAPContext { return true; } } - + return false; } http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java index 3b5c3cf..5fa34f7 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java @@ -90,7 +90,8 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel { final JoinInfo joinInfo = JoinInfo.of(left, right, condition); assert joinInfo.isEqui(); try { - return new OLAPJoinRel(getCluster(), traitSet, left, right, condition, joinInfo.leftKeys, joinInfo.rightKeys, variablesSet, joinType); + return new OLAPJoinRel(getCluster(), traitSet, left, right, condition, joinInfo.leftKeys, + joinInfo.rightKeys, variablesSet, joinType); } catch (InvalidRelException e) { // Semantic error not possible. Must be a bug. Convert to internal error. throw new AssertionError(e); @@ -175,11 +176,14 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel { this.context.allColumns.clear(); // build JoinDesc + Preconditions.checkState(this.getCondition() instanceof RexCall, "Cartesian Join is not supported."); + RexCall condition = (RexCall) this.getCondition(); JoinDesc join = buildJoin(condition); JoinRelType joinRelType = this.getJoinType(); - String joinType = joinRelType == JoinRelType.INNER ? "INNER" : joinRelType == JoinRelType.LEFT ? "LEFT" : null; + String joinType = joinRelType == JoinRelType.INNER ? "INNER" + : joinRelType == JoinRelType.LEFT ? "LEFT" : null; join.setType(joinType); this.context.joins.add(join); @@ -210,7 +214,8 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel { columns.addAll(rightColumnRowType.getAllColumns()); if (columns.size() != this.rowType.getFieldCount()) { - throw new IllegalStateException("RowType=" + this.rowType.getFieldCount() + ", ColumnRowType=" + columns.size()); + throw new IllegalStateException( + "RowType=" + this.rowType.getFieldCount() + ", ColumnRowType=" + columns.size()); } return new ColumnRowType(columns); } @@ -308,7 +313,8 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel { PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray()); RelOptTable factTable = context.firstTableScan.getTable(); - MethodCallExpression exprCall = Expressions.call(factTable.getExpression(OLAPTable.class), "executeOLAPQuery", implementor.getRootExpression(), Expressions.constant(context.id)); + MethodCallExpression exprCall = Expressions.call(factTable.getExpression(OLAPTable.class), "executeOLAPQuery", + implementor.getRootExpression(), Expressions.constant(context.id)); return implementor.result(physType, Blocks.toBlock(exprCall)); } http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java b/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java index dbf69bc..e96eee8 100644 --- a/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java +++ b/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java @@ -41,7 +41,8 @@ public class QueryRouter { private static final Logger logger = LoggerFactory.getLogger(QueryRouter.class); - public static IRealization selectRealization(OLAPContext olapContext, Set<IRealization> realizations) throws NoRealizationFoundException { + public static IRealization selectRealization(OLAPContext olapContext, Set<IRealization> realizations) + throws NoRealizationFoundException { String factTableName = olapContext.firstTableScan.getTableName(); String projectName = olapContext.olapSchema.getProjectName(); @@ -53,11 +54,16 @@ public class QueryRouter { candidates.add(new Candidate(real, sqlDigest)); } - logger.info("Find candidates by table " + factTableName + " and project=" + projectName + " : " + StringUtils.join(candidates, ",")); + logger.info("Find candidates by table " + factTableName + " and project=" + projectName + " : " + + StringUtils.join(candidates, ",")); + + List<Candidate> originCandidates = Lists.newArrayList(candidates); // rule based realization selection, rules might reorder realizations or remove specific realization RoutingRule.applyRules(candidates); + collectIncapableReason(olapContext, originCandidates); + if (candidates.size() == 0) { return null; } @@ -65,7 +71,8 @@ public class QueryRouter { Candidate chosen = candidates.get(0); adjustForDimensionAsMeasure(chosen, olapContext); - logger.info("The realizations remaining: " + RoutingRule.getPrintableText(candidates) + " And the final chosen one is the first one"); + logger.info("The realizations remaining: " + RoutingRule.getPrintableText(candidates) + + " And the final chosen one is the first one"); for (CapabilityInfluence influence : chosen.getCapability().influences) { if (influence.getInvolvedMeasure() != null) { @@ -88,4 +95,16 @@ public class QueryRouter { } } + private static void collectIncapableReason(OLAPContext olapContext, List<Candidate> candidates) { + for (Candidate candidate : candidates) { + if (!candidate.getCapability().capable) { + RealizationCheck.IncapableReason reason = RealizationCheck.IncapableReason + .create(candidate.getCapability().incapableCause); + if (reason != null) + olapContext.realizationCheck.addIncapableCube(candidate.getRealization(), reason); + } else { + olapContext.realizationCheck.addCapableCube(candidate.getRealization()); + } + } + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/query/src/main/java/org/apache/kylin/query/routing/RealizationCheck.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/routing/RealizationCheck.java b/query/src/main/java/org/apache/kylin/query/routing/RealizationCheck.java new file mode 100644 index 0000000..7556b83 --- /dev/null +++ b/query/src/main/java/org/apache/kylin/query/routing/RealizationCheck.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.query.routing; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import org.apache.kylin.cube.CubeInstance; +import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.metadata.model.DataModelDesc; +import org.apache.kylin.metadata.model.FunctionDesc; +import org.apache.kylin.metadata.model.TblColRef; +import org.apache.kylin.metadata.realization.CapabilityResult; +import org.apache.kylin.metadata.realization.IRealization; +import org.apache.kylin.query.relnode.OLAPTableScan; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +public class RealizationCheck { + private Map<DataModelDesc, List<IncapableReason>> modelIncapableReasons = Maps.newHashMap(); + private Map<CubeDesc, IncapableReason> cubeIncapableReasons = Maps.newHashMap(); + private Map<CubeDesc, Boolean> cubeCapabilities = Maps.newHashMap(); + private List<DataModelDesc> capableModels = Lists.newArrayList(); + + public Map<DataModelDesc, List<IncapableReason>> getModelIncapableReasons() { + return modelIncapableReasons; + } + + public Map<CubeDesc, IncapableReason> getCubeIncapableReasons() { + return cubeIncapableReasons; + } + + public Map<CubeDesc, Boolean> getCubeCapabilities() { + return cubeCapabilities; + } + + public void addCapableCube(IRealization realization) { + if (realization instanceof CubeInstance) { + cubeCapabilities.put(((CubeInstance) realization).getDescriptor(), true); + } + } + + public void addIncapableCube(IRealization realization) { + if (realization instanceof CubeInstance) { + cubeCapabilities.put(((CubeInstance) realization).getDescriptor(), false); + } + } + + public void addIncapableCube(IRealization realization, IncapableReason incapableReason) { + if (realization instanceof CubeInstance) { + cubeCapabilities.put(((CubeInstance) realization).getDescriptor(), false); + cubeIncapableReasons.put(((CubeInstance) realization).getDescriptor(), incapableReason); + } + } + + public void addCubeIncapableReason(IRealization realization, IncapableReason incapableReason) { + if (realization instanceof CubeInstance) { + cubeIncapableReasons.put(((CubeInstance) realization).getDescriptor(), incapableReason); + } + } + + public List<DataModelDesc> getCapableModels() { + return capableModels; + } + + public void addModelIncapableReason(DataModelDesc modelDesc, IncapableReason reason) { + if (!modelIncapableReasons.containsKey(modelDesc)) { + List<IncapableReason> reasons = Lists.newArrayList(reason); + modelIncapableReasons.put(modelDesc, reasons); + } else { + modelIncapableReasons.get(modelDesc).add(reason); + } + } + + public void addCapableModel(DataModelDesc modelDesc) { + this.capableModels.add(modelDesc); + } + + public void addModelIncapableReason(DataModelDesc modelDesc, List<IncapableReason> reasons) { + modelIncapableReasons.put(modelDesc, reasons); + } + + public static enum IncapableType { + CUBE_NOT_READY, CUBE_NOT_CONTAIN_TABLE, CUBE_NOT_CONTAIN_ALL_COLUMN, CUBE_NOT_CONTAIN_ALL_DIMENSION, CUBE_NOT_CONTAIN_ALL_MEASURE, CUBE_BLACK_OUT_REALIZATION, CUBE_UN_SUPPORT_MASSIN, CUBE_UN_SUPPORT_RAWQUERY, CUBE_UNMATCHED_DIMENSION, CUBE_LIMIT_PRECEDE_AGGR, CUBE_UNMATCHED_AGGREGATION, CUBE_OTHER_CUBE_INCAPABLE, MODEL_UNMATCHED_JOIN, MODEL_JOIN_TYPE_UNMATCHED, MODEL_JOIN_CONDITION_UNMATCHED, MODEL_JOIN_NOT_FOUND, MODEL_BAD_JOIN_SEQUENCE, MODEL_FACT_TABLE_NOT_FOUND, MODEL_OTHER_MODEL_INCAPABLE + } + + public static class IncapableReason { + private IncapableType incapableType; + // notFoundColumns = notFoundDimensions + notFoundMeasures; + private Collection<TblColRef> notFoundColumns; + private Collection<TblColRef> notFoundDimensions; + private Collection<TblColRef> notFoundMeasures; + private Collection<TblColRef> unmatchedDimensions; + private Collection<FunctionDesc> unmatchedAggregations; + private Collection<OLAPTableScan> notFoundTables; + + public static IncapableReason create(IncapableType incapableType) { + IncapableReason incapableReason = new IncapableReason(); + incapableReason.setIncapableType(incapableType); + return incapableReason; + } + + public static IncapableReason create(CapabilityResult.IncapableCause incapableCause) { + if (incapableCause == null) { + return null; + } + IncapableReason incapableReason = new IncapableReason(); + IncapableType incapableType = null; + switch (incapableCause.getIncapableType()) { + case UNSUPPORT_MASSIN: + incapableType = IncapableType.CUBE_UN_SUPPORT_MASSIN; + break; + case UNMATCHED_DIMENSION: + incapableType = IncapableType.CUBE_UNMATCHED_DIMENSION; + break; + case LIMIT_PRECEDE_AGGR: + incapableType = IncapableType.CUBE_LIMIT_PRECEDE_AGGR; + break; + case UNMATCHED_AGGREGATION: + incapableType = IncapableType.CUBE_UNMATCHED_AGGREGATION; + break; + case UNSUPPORT_RAWQUERY: + incapableType = IncapableType.CUBE_UN_SUPPORT_RAWQUERY; + break; + case II_UNMATCHED_FACT_TABLE: + incapableType = IncapableType.MODEL_FACT_TABLE_NOT_FOUND; + break; + case II_MISSING_COLS: + incapableType = IncapableType.CUBE_NOT_CONTAIN_ALL_COLUMN; + break; + default: + break; + } + incapableReason.setIncapableType(incapableType); + incapableReason.setUnmatchedDimensions(incapableCause.getUnmatchedDimensions()); + incapableReason.setUnmatchedAggregations(incapableCause.getUnmatchedAggregations()); + return incapableReason; + } + + public static IncapableReason notContainAllColumn(Collection<TblColRef> notFoundColumns) { + IncapableReason incapableReason = new IncapableReason(); + incapableReason.setIncapableType(IncapableType.CUBE_NOT_CONTAIN_ALL_COLUMN); + incapableReason.setNotFoundColumns(notFoundColumns); + return incapableReason; + } + + public static IncapableReason notContainAllDimension(Collection<TblColRef> notFoundDimensions) { + IncapableReason incapableReason = new IncapableReason(); + incapableReason.setIncapableType(IncapableType.CUBE_NOT_CONTAIN_ALL_DIMENSION); + incapableReason.setNotFoundDimensions(notFoundDimensions); + return incapableReason; + } + + public static IncapableReason notContainAllMeasures(Collection<TblColRef> notFoundMeasures) { + IncapableReason incapableReason = new IncapableReason(); + incapableReason.setIncapableType(IncapableType.CUBE_NOT_CONTAIN_ALL_MEASURE); + incapableReason.setNotFoundMeasures(notFoundMeasures); + return incapableReason; + } + + public static IncapableReason notFoundTables(Collection<OLAPTableScan> notFoundTables) { + IncapableReason incapableReason = new IncapableReason(); + incapableReason.setIncapableType(IncapableType.CUBE_NOT_CONTAIN_TABLE); + incapableReason.setNotFoundTables(notFoundTables); + return incapableReason; + } + + public void setIncapableType(IncapableType incapableType) { + this.incapableType = incapableType; + } + + public void setUnmatchedDimensions(Collection<TblColRef> unmatchedDimensions) { + this.unmatchedDimensions = unmatchedDimensions; + } + + public void setUnmatchedAggregations(Collection<FunctionDesc> unmatchedAggregations) { + this.unmatchedAggregations = unmatchedAggregations; + } + + public void setNotFoundColumns(Collection<TblColRef> notFoundColumns) { + this.notFoundColumns = notFoundColumns; + } + + public void setNotFoundTables(Collection<OLAPTableScan> notFoundTables) { + this.notFoundTables = notFoundTables; + } + + public void setNotFoundDimensions(Collection<TblColRef> notFoundDimensions) { + this.notFoundDimensions = notFoundDimensions; + } + + public void setNotFoundMeasures(Collection<TblColRef> notFoundMeasures) { + this.notFoundMeasures = notFoundMeasures; + } + + public Collection<TblColRef> getNotFoundDimensions() { + return notFoundDimensions; + } + + public Collection<TblColRef> getNotFoundMeasures() { + return notFoundMeasures; + } + + public IncapableType getIncapableType() { + return incapableType; + } + + public Collection<TblColRef> getUnmatchedDimensions() { + return unmatchedDimensions; + } + + public Collection<TblColRef> getNotFoundColumns() { + return notFoundColumns; + } + + public Collection<FunctionDesc> getUnmatchedAggregations() { + return unmatchedAggregations; + } + + public Collection<OLAPTableScan> getNotFoundTables() { + return notFoundTables; + } + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/query/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java b/query/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java index cfb0cbd..56580c6 100644 --- a/query/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java +++ b/query/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java @@ -43,6 +43,7 @@ import org.slf4j.LoggerFactory; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -53,7 +54,9 @@ public class RealizationChooser { // select models for given contexts, return realization candidates for each context public static void selectRealization(List<OLAPContext> contexts) { // try different model for different context + for (OLAPContext ctx : contexts) { + ctx.realizationCheck = new RealizationCheck(); attemptSelectRealization(ctx); Preconditions.checkNotNull(ctx.realization); } @@ -109,6 +112,8 @@ public class RealizationChooser { matchUp = ImmutableMap.of(firstTable.getAlias(), modelAlias); } else if (ctx.joins.size() != ctx.allTableScans.size() - 1) { // has hanging tables + ctx.realizationCheck.addModelIncapableReason(model, + RealizationCheck.IncapableReason.create(RealizationCheck.IncapableType.MODEL_BAD_JOIN_SEQUENCE)); throw new IllegalStateException("Please adjust the sequence of join tables. " + toErrorMsg(ctx)); } else { // normal big joins @@ -118,9 +123,13 @@ public class RealizationChooser { matchUp = ctx.joinsTree.matches(model.getJoinsTree(), result); } - if (matchUp == null) + if (matchUp == null) { + ctx.realizationCheck.addModelIncapableReason(model, + RealizationCheck.IncapableReason.create(RealizationCheck.IncapableType.MODEL_UNMATCHED_JOIN)); return null; + } + ctx.realizationCheck.addCapableModel(model); result.putAll(matchUp); return result; @@ -131,17 +140,28 @@ public class RealizationChooser { KylinConfig kylinConfig = first.olapSchema.getConfig(); String projectName = first.olapSchema.getProjectName(); String factTableName = first.firstTableScan.getOlapTable().getTableName(); - Set<IRealization> realizations = ProjectManager.getInstance(kylinConfig).getRealizationsByTable(projectName, factTableName); + Set<IRealization> realizations = ProjectManager.getInstance(kylinConfig).getRealizationsByTable(projectName, + factTableName); final Map<DataModelDesc, Set<IRealization>> models = Maps.newHashMap(); final Map<DataModelDesc, RealizationCost> costs = Maps.newHashMap(); + for (IRealization real : realizations) { - if (real.isReady() == false) + if (real.isReady() == false) { + context.realizationCheck.addIncapableCube(real, + RealizationCheck.IncapableReason.create(RealizationCheck.IncapableType.CUBE_NOT_READY)); continue; - if (containsAll(real.getAllColumnDescs(), first.allColumns) == false) + } + if (containsAll(real.getAllColumnDescs(), first.allColumns) == false) { + context.realizationCheck.addIncapableCube(real, RealizationCheck.IncapableReason + .notContainAllColumn(notContain(real.getAllColumnDescs(), first.allColumns))); continue; - if (RemoveBlackoutRealizationsRule.accept(real) == false) + } + if (RemoveBlackoutRealizationsRule.accept(real) == false) { + context.realizationCheck.addIncapableCube(real, RealizationCheck.IncapableReason + .create(RealizationCheck.IncapableType.CUBE_BLACK_OUT_REALIZATION)); continue; + } RealizationCost cost = new RealizationCost(real); DataModelDesc m = real.getModel(); @@ -184,6 +204,15 @@ public class RealizationChooser { return true; } + private static List<TblColRef> notContain(Set<ColumnDesc> allColumnDescs, Set<TblColRef> allColumns) { + List<TblColRef> notContainCols = Lists.newArrayList(); + for (TblColRef col : allColumns) { + if (!allColumnDescs.contains(col.getColumnDesc())) + notContainCols.add(col); + } + return notContainCols; + } + private static void fixModel(OLAPContext context, DataModelDesc model, Map<String, String> aliasMap) { for (OLAPTableScan tableScan : context.allTableScans) { tableScan.fixColumnRowTypeWithModel(model, aliasMap); @@ -205,7 +234,8 @@ public class RealizationChooser { this.priority = Candidate.PRIORITIES.get(real.getType()); // ref CubeInstance.getCost() - int c = real.getAllDimensions().size() * CubeInstance.COST_WEIGHT_DIMENSION + real.getMeasures().size() * CubeInstance.COST_WEIGHT_MEASURE; + int c = real.getAllDimensions().size() * CubeInstance.COST_WEIGHT_DIMENSION + + real.getMeasures().size() * CubeInstance.COST_WEIGHT_MEASURE; for (JoinTableDesc join : real.getModel().getJoinTables()) { if (join.getJoin().isInnerJoin()) c += CubeInstance.COST_WEIGHT_INNER_JOIN; http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveUncapableRealizationsRule.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveUncapableRealizationsRule.java b/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveUncapableRealizationsRule.java index 576b47f..56e6eb5 100644 --- a/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveUncapableRealizationsRule.java +++ b/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveUncapableRealizationsRule.java @@ -34,9 +34,8 @@ public class RemoveUncapableRealizationsRule extends RoutingRule { Candidate candidate = iterator.next(); CapabilityResult capability = candidate.getRealization().isCapable(candidate.getSqlDigest()); - if (capability.capable) - candidate.setCapability(capability); - else + candidate.setCapability(capability); + if (!capability.capable) iterator.remove(); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java index d46d4ff..377ca89 100644 --- a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java +++ b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java @@ -186,10 +186,10 @@ public class QueryUtil { errorMsg = errorMsg.replaceAll("\\s", " "); // move cause to be ahead of sql, calcite creates the message pattern below - Pattern pattern = Pattern.compile("error while executing SQL \"(.*)\":(.*)"); + Pattern pattern = Pattern.compile("Error while executing SQL \"(.*)\":(.*)"); Matcher matcher = pattern.matcher(errorMsg); if (matcher.find()) { - return matcher.group(2).trim() + "\n" + "while executing SQL: \"" + matcher.group(1).trim() + "\""; + return matcher.group(2).trim() + "\nwhile executing SQL: \"" + matcher.group(1).trim() + "\""; } else return errorMsg; } catch (Exception e) { @@ -201,7 +201,8 @@ public class QueryUtil { String sql1 = sql.toLowerCase(); sql1 = removeCommentInSql(sql1); sql1 = sql1.trim(); - return sql1.startsWith("select") || (sql1.startsWith("with") && sql1.contains("select")) || (sql1.startsWith("explain") && sql1.contains("select")); + return sql1.startsWith("select") || (sql1.startsWith("with") && sql1.contains("select")) + || (sql1.startsWith("explain") && sql1.contains("select")); } public static String removeCommentInSql(String sql1) { http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java index 2b1626f..088ed6d 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java @@ -456,7 +456,7 @@ public class QueryService extends BasicService { } catch (Throwable e) { // calcite may throw AssertError logger.error("Exception while executing query", e); - String errMsg = QueryUtil.makeErrorMsgUserFriendly(e); + String errMsg = makeErrorMsgUserFriendly(e); sqlResponse = new SQLResponse(null, null, 0, true, errMsg); sqlResponse.setTotalScanCount(queryContext.getScannedRows()); @@ -871,6 +871,10 @@ public class QueryService extends BasicService { return buildSqlResponse(isPushDown, results, columnMetas); } + protected String makeErrorMsgUserFriendly(Throwable e) { + return QueryUtil.makeErrorMsgUserFriendly(e); + } + private SQLResponse getPrepareOnlySqlResponse(String correctedSql, Connection conn, Boolean isPushDown, List<List<String>> results, List<SelectedColumnMeta> columnMetas) throws SQLException { http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java index c9f9296..7c5f253 100644 --- a/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java +++ b/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java @@ -69,10 +69,11 @@ public class QueryControllerTest extends ServiceTestBase { @Test public void testErrorMsg() { - String errorMsg = "error while executing SQL \"select lkp.clsfd_ga_prfl_id, ga.sum_dt, sum(ga.bounces) as bounces, sum(ga.exits) as exits, sum(ga.entrances) as entrances, sum(ga.pageviews) as pageviews, count(distinct ga.GA_VSTR_ID, ga.GA_VST_ID) as visits, count(distinct ga.GA_VSTR_ID) as uniqVistors from CLSFD_GA_PGTYPE_CATEG_LOC ga left join clsfd_ga_prfl_lkp lkp on ga.SRC_GA_PRFL_ID = lkp.SRC_GA_PRFL_ID group by lkp.clsfd_ga_prfl_id,ga.sum_dt order by lkp.clsfd_ga_prfl_id,ga.sum_dt LIMIT 50000\": From line 14, column 14 to line 14, column 29: Column 'CLSFD_GA_PRFL_ID' not found in table 'LKP'"; - assert QueryUtil.makeErrorMsgUserFriendly(errorMsg).equals( + String errorMsg = "Error while executing SQL \"select lkp.clsfd_ga_prfl_id, ga.sum_dt, sum(ga.bounces) as bounces, sum(ga.exits) as exits, sum(ga.entrances) as entrances, sum(ga.pageviews) as pageviews, count(distinct ga.GA_VSTR_ID, ga.GA_VST_ID) as visits, count(distinct ga.GA_VSTR_ID) as uniqVistors from CLSFD_GA_PGTYPE_CATEG_LOC ga left join clsfd_ga_prfl_lkp lkp on ga.SRC_GA_PRFL_ID = lkp.SRC_GA_PRFL_ID group by lkp.clsfd_ga_prfl_id,ga.sum_dt order by lkp.clsfd_ga_prfl_id,ga.sum_dt LIMIT 50000\": From line 14, column 14 to line 14, column 29: Column 'CLSFD_GA_PRFL_ID' not found in table 'LKP'"; + Assert.assertEquals( "From line 14, column 14 to line 14, column 29: Column 'CLSFD_GA_PRFL_ID' not found in table 'LKP'\n" - + "while executing SQL: \"select lkp.clsfd_ga_prfl_id, ga.sum_dt, sum(ga.bounces) as bounces, sum(ga.exits) as exits, sum(ga.entrances) as entrances, sum(ga.pageviews) as pageviews, count(distinct ga.GA_VSTR_ID, ga.GA_VST_ID) as visits, count(distinct ga.GA_VSTR_ID) as uniqVistors from CLSFD_GA_PGTYPE_CATEG_LOC ga left join clsfd_ga_prfl_lkp lkp on ga.SRC_GA_PRFL_ID = lkp.SRC_GA_PRFL_ID group by lkp.clsfd_ga_prfl_id,ga.sum_dt order by lkp.clsfd_ga_prfl_id,ga.sum_dt LIMIT 50000\""); + + "while executing SQL: \"select lkp.clsfd_ga_prfl_id, ga.sum_dt, sum(ga.bounces) as bounces, sum(ga.exits) as exits, sum(ga.entrances) as entrances, sum(ga.pageviews) as pageviews, count(distinct ga.GA_VSTR_ID, ga.GA_VST_ID) as visits, count(distinct ga.GA_VSTR_ID) as uniqVistors from CLSFD_GA_PGTYPE_CATEG_LOC ga left join clsfd_ga_prfl_lkp lkp on ga.SRC_GA_PRFL_ID = lkp.SRC_GA_PRFL_ID group by lkp.clsfd_ga_prfl_id,ga.sum_dt order by lkp.clsfd_ga_prfl_id,ga.sum_dt LIMIT 50000\"", + QueryUtil.makeErrorMsgUserFriendly(errorMsg)); } @Test
