KYLIN-2575 code refactors
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/aed840f9 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/aed840f9 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/aed840f9 Branch: refs/heads/master Commit: aed840f9343e3e62a711517900c357a091e75619 Parents: 8906d13 Author: Hongbin Ma <mahong...@apache.org> Authored: Mon Aug 21 11:51:15 2017 +0800 Committer: Hongbin Ma <m...@kyligence.io> Committed: Mon Aug 21 22:55:59 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 18 +- .../common/util/CaseInsensitiveString.java | 63 +++++++ .../kylin/common/util/CliCommandExecutor.java | 5 +- .../org/apache/kylin/common/util/BasicTest.java | 8 +- .../cube/cuboid/DefaultCuboidScheduler.java | 18 +- .../org/apache/kylin/cube/model/CubeDesc.java | 92 ++++++---- .../cube/model/TooManyCuboidException.java | 25 +++ .../org/apache/kylin/cube/CubeDescTest.java | 7 +- .../org/apache/kylin/job/JoinedFlatTable.java | 2 +- .../apache/kylin/metadata/model/ColumnDesc.java | 15 +- .../metadata/model/ComputedColumnDesc.java | 73 ++++++-- .../kylin/metadata/model/DataModelDesc.java | 21 ++- .../metadata/model/tool/CalciteParser.java | 102 ++--------- .../kylin/metadata/project/ProjectManager.java | 9 +- .../kylin/model/tool/CalciteParserTest.java | 16 -- .../gtrecord/GTCubeStorageQueryBase.java | 54 ++++-- .../localmeta/cube_desc/ci_inner_join_cube.json | 46 +++-- .../localmeta/cube_desc/ci_left_join_cube.json | 46 +++-- .../cube_desc/ut_inner_join_cube_partial.json | 28 +-- .../model_desc/ci_inner_join_model.json | 52 ++++-- .../model_desc/ci_left_join_model.json | 54 ++++-- .../query/sql_computedcolumn/query01.sql | 2 +- .../query/sql_computedcolumn/query02.sql | 2 +- .../query/sql_computedcolumn/query03.sql | 2 +- .../query/sql_verifyCount/query01.sql.expected | 2 +- .../query/sql_verifyCount/query03.sql.expected | 2 +- .../query/sql_verifyCount/query04.sql.expected | 2 +- .../query/sql_verifyCount/query10.sql.expected | 2 +- .../query/sql_verifyCount/query11.sql.expected | 2 +- .../query/adhoc/PushDownRunnerJdbcImpl.java | 2 +- .../query/enumerator/LookupTableEnumerator.java | 2 +- .../kylin/query/relnode/ColumnRowType.java | 4 + .../kylin/query/relnode/OLAPProjectRel.java | 2 +- .../kylin/query/relnode/OLAPTableScan.java | 2 +- .../apache/kylin/query/schema/OLAPSchema.java | 12 +- .../kylin/query/schema/OLAPSchemaFactory.java | 2 +- .../apache/kylin/query/schema/OLAPTable.java | 22 ++- .../query/util/KeywordDefaultDirtyHack.java | 2 +- .../apache/kylin/query/util/PushDownUtil.java | 174 ++++++++++--------- .../org/apache/kylin/query/util/QueryUtil.java | 25 ++- .../kylin/query/util/PushDownUtilTest.java | 43 ++++- .../apache/kylin/query/util/QueryUtilTest.java | 10 +- .../rest/controller2/ModelControllerV2.java | 34 +++- .../apache/kylin/rest/service/ModelService.java | 113 +++++++++++- .../apache/kylin/rest/service/QueryService.java | 75 ++++---- 45 files changed, 861 insertions(+), 433 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index df74f5e..0f5f3a9 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -605,7 +605,7 @@ abstract public class KylinConfigBase implements Serializable { public Map<String, String> getKafkaConfigOverride() { return getPropertiesByPrefix("kylin.source.kafka.config-override."); } - + // ============================================================================ // SOURCE.JDBC // ============================================================================ @@ -613,23 +613,23 @@ abstract public class KylinConfigBase implements Serializable { public String getJdbcConnectionUrl() { return getOptional("kylin.source.jdbc.connection-url"); } - + public String getJdbcDriver() { return getOptional("kylin.source.jdbc.driver"); } - + public String getJdbcDialect() { return getOptional("kylin.source.jdbc.dialect"); } - + public String getJdbcUser() { return getOptional("kylin.source.jdbc.user"); } - + public String getJdbcPass() { return getOptional("kylin.source.jdbc.pass"); } - + public String getSqoopHome() { return getOptional("kylin.source.jdbc.sqoop-home"); } @@ -1042,9 +1042,9 @@ abstract public class KylinConfigBase implements Serializable { return getOptional("kylin.query.pushdown.runner-class-name", ""); } - public String getPushDownConverterClassName() { - return getOptional("kylin.query.pushdown.converter-class-name", - "org.apache.kylin.source.adhocquery.HivePushDownConverter"); + public String[] getPushDownConverterClassNames() { + return getOptionalStringArray("kylin.query.pushdown.converter-class-names", + new String[] { "org.apache.kylin.source.adhocquery.HivePushDownConverter" }); } public String getJdbcUrl() { http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/core-common/src/main/java/org/apache/kylin/common/util/CaseInsensitiveString.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/CaseInsensitiveString.java b/core-common/src/main/java/org/apache/kylin/common/util/CaseInsensitiveString.java new file mode 100644 index 0000000..4389aab --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/util/CaseInsensitiveString.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kylin.common.util; + +/** + * A string wrapper that makes .equals a caseInsensitive match + * <p> + * a collection that wraps a String mapping in CaseInsensitiveStrings will still accept a String but will now + * return a caseInsensitive match rather than a caseSensitive one + * </p> + */ +public class CaseInsensitiveString { + String str; + + private CaseInsensitiveString(String str) { + this.str = str; + } + + public static CaseInsensitiveString wrap(String str) { + return new CaseInsensitiveString(str); + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null) + return false; + + if (o.getClass() == getClass()) { //is another CaseInsensitiveString + CaseInsensitiveString that = (CaseInsensitiveString) o; + return (str != null) ? str.equalsIgnoreCase(that.str) : that.str == null; + } else { + return false; + } + + } + + @Override + public int hashCode() { + return (str != null) ? str.toUpperCase().hashCode() : 0; + } + + @Override + public String toString() { + return str; + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java b/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java index 9d41c86..0b6ac90 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java @@ -89,9 +89,10 @@ public class CliCommandExecutor { } if (r.getFirst() != 0) - throw new IOException("OS command error exit with " + r.getFirst() // + throw new IOException("OS command error exit with return code: " + r.getFirst() // + + ", error message: " + r.getSecond() + "The command is: \n" + command + (remoteHost == null ? "" : " (remoteHost:" + remoteHost + ")") // - + " -- " + command + "\n" + r.getSecond()); + ); return r; } http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java index 5512147b..fe89f24 100644 --- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java @@ -81,6 +81,8 @@ public class BasicTest { @Test public void testxx() throws InterruptedException { + System.out.println( + "((?<![\\p{L}_0-9\\.\\\"])(\\\"[\\p{L}_0-9]+\\\"\\.)?(\\\"[\\p{L}_0-9]+\\\")(?![\\p{L}_0-9\\.\\\"]))"); System.out.println(0x8fL); byte[] space = new byte[100]; ByteBuffer buffer = ByteBuffer.wrap(space, 10, 20); @@ -190,8 +192,10 @@ public class BasicTest { System.out.println(time(c.getTimeInMillis())); a.setTimeInMillis(current); - b.set(a.get(Calendar.YEAR), a.get(Calendar.MONTH), a.get(Calendar.DAY_OF_MONTH), a.get(Calendar.HOUR_OF_DAY), a.get(Calendar.MINUTE)); - c.set(a.get(Calendar.YEAR), a.get(Calendar.MONTH), a.get(Calendar.DAY_OF_MONTH), a.get(Calendar.HOUR_OF_DAY), 0); + b.set(a.get(Calendar.YEAR), a.get(Calendar.MONTH), a.get(Calendar.DAY_OF_MONTH), a.get(Calendar.HOUR_OF_DAY), + a.get(Calendar.MINUTE)); + c.set(a.get(Calendar.YEAR), a.get(Calendar.MONTH), a.get(Calendar.DAY_OF_MONTH), a.get(Calendar.HOUR_OF_DAY), + 0); System.out.println(time(b.getTimeInMillis())); System.out.println(time(c.getTimeInMillis())); http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java index 1c18b69..f7d22da 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java @@ -36,6 +36,7 @@ import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.Pair; import org.apache.kylin.cube.model.AggregationGroup; import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.cube.model.TooManyCuboidException; import com.google.common.base.Predicate; import com.google.common.collect.Iterators; @@ -47,13 +48,13 @@ public class DefaultCuboidScheduler extends CuboidScheduler { private final long max; private final Set<Long> allCuboidIds; private final Map<Long, List<Long>> parent2child; - + public DefaultCuboidScheduler(CubeDesc cubeDesc) { super(cubeDesc); - + int size = this.cubeDesc.getRowkey().getRowKeyColumns().length; this.max = (long) Math.pow(2, size) - 1; - + Pair<Set<Long>, Map<Long, List<Long>>> pair = buildTreeBottomUp(); this.allCuboidIds = pair.getFirst(); this.parent2child = pair.getSecond(); @@ -145,7 +146,8 @@ public class DefaultCuboidScheduler extends CuboidScheduler { maxCombination = maxCombination < 0 ? Long.MAX_VALUE : maxCombination; while (!children.isEmpty()) { if (cuboidHolder.size() > maxCombination) { - throw new IllegalStateException("Too many cuboids for the cube. Cuboid combination reached " + cuboidHolder.size() + " and limit is " + maxCombination + ". Abort calculation."); + throw new IllegalStateException("Too many cuboids for the cube. Cuboid combination reached " + + cuboidHolder.size() + " and limit is " + maxCombination + ". Abort calculation."); } cuboidHolder.addAll(children); children = getOnTreeParentsByLayer(children); @@ -251,7 +253,7 @@ public class DefaultCuboidScheduler extends CuboidScheduler { return parentCandidate; } - + /** * Get all valid cuboids for agg group, ignoring padding * @param agg agg group @@ -265,7 +267,7 @@ public class DefaultCuboidScheduler extends CuboidScheduler { Set<Long> children = getLowestCuboids(agg); while (!children.isEmpty()) { if (cuboidHolder.size() > cubeDesc.getConfig().getCubeAggrGroupMaxCombination()) { - throw new IllegalStateException("too many combination for the aggregation group"); + throw new TooManyCuboidException("Holder size larger than kylin.cube.aggrgroup.max-combination"); } cuboidHolder.addAll(children); children = getOnTreeParentsByLayer(children, agg); @@ -341,12 +343,12 @@ public class DefaultCuboidScheduler extends CuboidScheduler { } return false; } - + private boolean checkDimCap(AggregationGroup agg, long cuboidID) { int dimCap = agg.getDimCap(); if (dimCap <= 0) return true; - + return Long.bitCount(cuboidID) <= dimCap; } http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 439e484..0e22587 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -114,7 +114,8 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { @Override public String toString() { - return "DeriveInfo [type=" + type + ", join=" + join + ", columns=" + Arrays.toString(columns) + ", isOneToOne=" + isOneToOne + "]"; + return "DeriveInfo [type=" + type + ", join=" + join + ", columns=" + Arrays.toString(columns) + + ", isOneToOne=" + isOneToOne + "]"; } } @@ -286,7 +287,8 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { throw new RuntimeException("Cannot get host info for " + derived); } - public Map<Array<TblColRef>, List<DeriveInfo>> getHostToDerivedInfo(List<TblColRef> rowCols, Collection<TblColRef> wantedCols) { + public Map<Array<TblColRef>, List<DeriveInfo>> getHostToDerivedInfo(List<TblColRef> rowCols, + Collection<TblColRef> wantedCols) { Map<Array<TblColRef>, List<DeriveInfo>> result = new HashMap<Array<TblColRef>, List<DeriveInfo>>(); for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedMap.entrySet()) { Array<TblColRef> hostCols = entry.getKey(); @@ -496,12 +498,15 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { KylinVersion cubeVersion = new KylinVersion(getVersion()); KylinVersion kylinVersion = KylinVersion.getCurrentVersion(); if (!kylinVersion.isCompatibleWith(cubeVersion)) { - logger.info("checkSignature on {} is skipped as the its version {} is different from kylin version {}", getName(), cubeVersion, kylinVersion); + logger.info("checkSignature on {} is skipped as the its version {} is different from kylin version {}", + getName(), cubeVersion, kylinVersion); return true; } if (kylinVersion.isCompatibleWith(cubeVersion) && !kylinVersion.isSignatureCompatibleWith(cubeVersion)) { - logger.info("checkSignature on {} is skipped as the its version is {} (not signature compatible but compatible) ", getName(), cubeVersion); + logger.info( + "checkSignature on {} is skipped as the its version is {} (not signature compatible but compatible) ", + getName(), cubeVersion); return true; } @@ -606,7 +611,9 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { // check all dimension columns are presented on rowkey List<TblColRef> dimCols = listDimensionColumnsExcludingDerived(true); - checkState(rowkey.getRowKeyColumns().length == dimCols.size(), "RowKey columns count (%s) doesn't match dimensions columns count (%s)", rowkey.getRowKeyColumns().length, dimCols.size()); + checkState(rowkey.getRowKeyColumns().length == dimCols.size(), + "RowKey columns count (%s) doesn't match dimensions columns count (%s)", + rowkey.getRowKeyColumns().length, dimCols.size()); initDictionaryDesc(); amendAllColumns(); @@ -615,7 +622,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { public CuboidScheduler getCuboidScheduler() { if (cuboidScheduler != null) return cuboidScheduler; - + synchronized (this) { if (cuboidScheduler == null) { cuboidScheduler = CuboidScheduler.getInstance(this); @@ -632,22 +639,25 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { int index = 0; for (AggregationGroup agg : getAggregationGroups()) { - long combination = 0L; try { - combination = agg.calculateCuboidCombination(); - } catch (Exception ex) { - combination = config.getCubeAggrGroupMaxCombination() + 1; - } finally { + long combination = agg.calculateCuboidCombination(); + if (combination > config.getCubeAggrGroupMaxCombination()) { - String msg = "Aggregation group " + index + " has too many combinations, use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max-combination' to a bigger value."; - logger.error("Aggregation group " + index + " has " + combination + " combinations;"); - logger.error(msg); - throw new IllegalStateException(msg); + String msg = "Aggregation group " + index + " of Cube Desc " + this.name + + " has too many combinations: " + combination + + ". Use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max-combination' to a bigger value."; + throw new TooManyCuboidException(msg); } + } catch (TooManyCuboidException tmce) { + throw tmce; + } catch (Exception e) { + throw new IllegalStateException("Unknown error while calculating cuboid number for " + // + "Aggregation group " + index + " of Cube Desc " + this.name, e); } index++; } + } public void validateAggregationGroups() { @@ -678,50 +688,64 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { Set<String> jointDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); getDims(jointDimsList, jointDims, agg.getSelectRule().jointDims); - if (!includeDims.containsAll(mandatoryDims) || !includeDims.containsAll(hierarchyDims) || !includeDims.containsAll(jointDims)) { + if (!includeDims.containsAll(mandatoryDims) || !includeDims.containsAll(hierarchyDims) + || !includeDims.containsAll(jointDims)) { List<String> notIncluded = Lists.newArrayList(); - final Iterable<String> all = Iterables.unmodifiableIterable(Iterables.concat(mandatoryDims, hierarchyDims, jointDims)); + final Iterable<String> all = Iterables + .unmodifiableIterable(Iterables.concat(mandatoryDims, hierarchyDims, jointDims)); for (String dim : all) { if (includeDims.contains(dim) == false) { notIncluded.add(dim); } } Collections.sort(notIncluded); - logger.error("Aggregation group " + index + " Include dimensions not containing all the used dimensions"); - throw new IllegalStateException("Aggregation group " + index + " 'includes' dimensions not include all the dimensions:" + notIncluded.toString()); + logger.error( + "Aggregation group " + index + " Include dimensions not containing all the used dimensions"); + throw new IllegalStateException("Aggregation group " + index + + " 'includes' dimensions not include all the dimensions:" + notIncluded.toString()); } if (CollectionUtils.containsAny(mandatoryDims, hierarchyDims)) { - logger.warn("Aggregation group " + index + " mandatory dimensions overlap with hierarchy dimensions: " + ensureOrder(CollectionUtils.intersection(mandatoryDims, hierarchyDims))); + logger.warn("Aggregation group " + index + " mandatory dimensions overlap with hierarchy dimensions: " + + ensureOrder(CollectionUtils.intersection(mandatoryDims, hierarchyDims))); } if (CollectionUtils.containsAny(mandatoryDims, jointDims)) { - logger.warn("Aggregation group " + index + " mandatory dimensions overlap with joint dimensions: " + ensureOrder(CollectionUtils.intersection(mandatoryDims, jointDims))); + logger.warn("Aggregation group " + index + " mandatory dimensions overlap with joint dimensions: " + + ensureOrder(CollectionUtils.intersection(mandatoryDims, jointDims))); } if (CollectionUtils.containsAny(hierarchyDims, jointDims)) { logger.error("Aggregation group " + index + " hierarchy dimensions overlap with joint dimensions"); - throw new IllegalStateException("Aggregation group " + index + " hierarchy dimensions overlap with joint dimensions: " + ensureOrder(CollectionUtils.intersection(hierarchyDims, jointDims))); + throw new IllegalStateException( + "Aggregation group " + index + " hierarchy dimensions overlap with joint dimensions: " + + ensureOrder(CollectionUtils.intersection(hierarchyDims, jointDims))); } if (hasSingleOrNone(hierarchyDimsList)) { logger.error("Aggregation group " + index + " require at least 2 dimensions in a hierarchy"); - throw new IllegalStateException("Aggregation group " + index + " require at least 2 dimensions in a hierarchy."); + throw new IllegalStateException( + "Aggregation group " + index + " require at least 2 dimensions in a hierarchy."); } if (hasSingleOrNone(jointDimsList)) { logger.error("Aggregation group " + index + " require at least 2 dimensions in a joint"); - throw new IllegalStateException("Aggregation group " + index + " require at least 2 dimensions in a joint"); + throw new IllegalStateException( + "Aggregation group " + index + " require at least 2 dimensions in a joint"); } Pair<Boolean, Set<String>> overlap = hasOverlap(hierarchyDimsList, hierarchyDims); if (overlap.getFirst() == true) { - logger.error("Aggregation group " + index + " a dimension exist in more than one hierarchy: " + ensureOrder(overlap.getSecond())); - throw new IllegalStateException("Aggregation group " + index + " a dimension exist in more than one hierarchy: " + ensureOrder(overlap.getSecond())); + logger.error("Aggregation group " + index + " a dimension exist in more than one hierarchy: " + + ensureOrder(overlap.getSecond())); + throw new IllegalStateException("Aggregation group " + index + + " a dimension exist in more than one hierarchy: " + ensureOrder(overlap.getSecond())); } overlap = hasOverlap(jointDimsList, jointDims); if (overlap.getFirst() == true) { - logger.error("Aggregation group " + index + " a dimension exist in more than one joint: " + ensureOrder(overlap.getSecond())); - throw new IllegalStateException("Aggregation group " + index + " a dimension exist in more than one joint: " + ensureOrder(overlap.getSecond())); + logger.error("Aggregation group " + index + " a dimension exist in more than one joint: " + + ensureOrder(overlap.getSecond())); + throw new IllegalStateException("Aggregation group " + index + + " a dimension exist in more than one joint: " + ensureOrder(overlap.getSecond())); } index++; @@ -849,7 +873,8 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { return new String[][] { cols, extra }; } - private void initDerivedMap(TblColRef[] hostCols, DeriveType type, JoinDesc join, TblColRef[] derivedCols, String[] extra) { + private void initDerivedMap(TblColRef[] hostCols, DeriveType type, JoinDesc join, TblColRef[] derivedCols, + String[] extra) { if (hostCols.length == 0 || derivedCols.length == 0) throw new IllegalStateException("host/derived columns must not be empty"); @@ -870,7 +895,8 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { for (int i = 0; i < derivedCols.length; i++) { TblColRef derivedCol = derivedCols[i]; - boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols, derivedCol) || (extra != null && extra[i].contains("1-1")); + boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols, derivedCol) + || (extra != null && extra[i].contains("1-1")); derivedToHostMap.put(derivedCol, new DeriveInfo(type, join, hostCols, isOneToOne)); } @@ -902,7 +928,8 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { whatsLeft.add(derCol); } if (whatsLeft.size() > 0) { - infoList.add(new DeriveInfo(type, join, (TblColRef[]) whatsLeft.toArray(new TblColRef[whatsLeft.size()]), false)); + infoList.add(new DeriveInfo(type, join, (TblColRef[]) whatsLeft.toArray(new TblColRef[whatsLeft.size()]), + false)); } } @@ -993,7 +1020,8 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { } for (int i = 0; i < measures.size(); i++) { - checkState(checkEachMeasureExist.get(i), "measure (%s) does not exist in column familyï¼or measure duplicates", measures.get(i)); + checkState(checkEachMeasureExist.get(i), + "measure (%s) does not exist in column familyï¼or measure duplicates", measures.get(i)); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/core-cube/src/main/java/org/apache/kylin/cube/model/TooManyCuboidException.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/TooManyCuboidException.java b/core-cube/src/main/java/org/apache/kylin/cube/model/TooManyCuboidException.java new file mode 100644 index 0000000..cfa41e4 --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/TooManyCuboidException.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kylin.cube.model; + +public class TooManyCuboidException extends RuntimeException { + + public TooManyCuboidException(String message) { + super(message); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java index 0bbc874..b8c19c4 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java @@ -44,6 +44,7 @@ import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; import org.apache.kylin.cube.model.CubeDesc.DeriveType; import org.apache.kylin.cube.model.DimensionDesc; import org.apache.kylin.cube.model.SelectRule; +import org.apache.kylin.cube.model.TooManyCuboidException; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.TblColRef; import org.junit.After; @@ -204,8 +205,8 @@ public class CubeDescTest extends LocalFileMetadataTestCase { @Test public void testBadInit4() throws Exception { - thrown.expect(IllegalStateException.class); - thrown.expectMessage("Aggregation group 0 has too many combinations, use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max-combination' to a bigger value."); + thrown.expect(TooManyCuboidException.class); + thrown.expectMessage("Aggregation group 0 of Cube Desc test_kylin_cube_with_slr_desc has too many combinations: 31. Use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max-combination' to a bigger value."); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); try { @@ -312,7 +313,7 @@ public class CubeDescTest extends LocalFileMetadataTestCase { } } - thrown.expect(IllegalStateException.class); + thrown.expect(TooManyCuboidException.class); CubeDescManager.clearCache(); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ut_cube_desc_combination_int_overflow"); cubeDesc.init(getTestConfig()); http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java index 5ea2335..acb29e1 100644 --- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java +++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java @@ -148,7 +148,7 @@ public class JoinedFlatTable { return sql.toString(); } - private static void appendJoinStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine) { + public static void appendJoinStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine) { final String sep = singleLine ? " " : "\n"; Set<TableRef> dimTableCache = new HashSet<>(); http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java index 3c84c96..394300e 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java @@ -21,7 +21,6 @@ package org.apache.kylin.metadata.model; import java.io.Serializable; import org.apache.kylin.metadata.datatype.DataType; -import org.apache.kylin.metadata.model.tool.CalciteParser; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; @@ -198,8 +197,7 @@ public class ColumnDesc implements Serializable { Preconditions.checkState(computedColumnExpr != null); Preconditions.checkState(tableAlias != null); - String s = CalciteParser.insertAliasInExpr(computedColumnExpr, tableAlias); - return s; + return computedColumnExpr; } public boolean isComputedColumnn() { @@ -223,6 +221,17 @@ public class ColumnDesc implements Serializable { } } + // for test mainly + public static ColumnDesc mockup(TableDesc table, int oneBasedColumnIndex, String name, String datatype) { + ColumnDesc desc = new ColumnDesc(); + String id = "" + oneBasedColumnIndex; + desc.setId(id); + desc.setName(name); + desc.setDatatype(datatype); + desc.init(table); + return desc; + } + @Override public int hashCode() { final int prime = 31; http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java index 10a7dd3..ab8a634 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java @@ -17,20 +17,30 @@ */ package org.apache.kylin.metadata.model; +import java.io.Serializable; +import java.util.Set; + +import org.apache.commons.lang.StringUtils; import org.apache.kylin.metadata.model.tool.CalciteParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import java.io.Serializable; - @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE) public class ComputedColumnDesc implements Serializable { + private static final Logger logger = LoggerFactory.getLogger(ComputedColumnDesc.class); + + @JsonProperty + private String tableIdentity; // the alias in the model where the computed column belong to @JsonProperty - private String tableIdentity; + @JsonInclude(JsonInclude.Include.NON_NULL) + private String tableAlias; @JsonProperty - private String columnName; + private String columnName; // the new col name @JsonProperty private String expression; @JsonProperty @@ -38,21 +48,56 @@ public class ComputedColumnDesc implements Serializable { @JsonProperty private String comment; - public void init() { + public void init(Set<String> aliasSet, String rootFactTableName) { Preconditions.checkNotNull(tableIdentity, "tableIdentity is null"); Preconditions.checkNotNull(columnName, "columnName is null"); Preconditions.checkNotNull(expression, "expression is null"); Preconditions.checkNotNull(datatype, "datatype is null"); - Preconditions.checkState(tableIdentity.equals(tableIdentity.trim()), "tableIdentity of ComputedColumnDesc has heading/tailing whitespace"); - Preconditions.checkState(columnName.equals(columnName.trim()), "columnName of ComputedColumnDesc has heading/tailing whitespace"); - Preconditions.checkState(datatype.equals(datatype.trim()), "datatype of ComputedColumnDesc has heading/tailing whitespace"); + if (tableAlias == null) // refer to comment of handleLegacyCC() + tableAlias = tableIdentity.substring(tableIdentity.indexOf(".") + 1); + + Preconditions.checkState(tableIdentity.equals(tableIdentity.trim()), + "tableIdentity of ComputedColumnDesc has heading/tailing whitespace"); + Preconditions.checkState(tableAlias.equals(tableAlias.trim()), + "tableAlias of ComputedColumnDesc has heading/tailing whitespace"); + Preconditions.checkState(columnName.equals(columnName.trim()), + "columnName of ComputedColumnDesc has heading/tailing whitespace"); + Preconditions.checkState(datatype.equals(datatype.trim()), + "datatype of ComputedColumnDesc has heading/tailing whitespace"); tableIdentity = tableIdentity.toUpperCase(); + tableAlias = tableAlias.toUpperCase(); columnName = columnName.toUpperCase(); - if ("true".equals(System.getProperty("needCheckCC"))) - CalciteParser.ensureNoTableNameExists(expression); + if (!tableIdentity.contains(rootFactTableName) || !tableAlias.equals(rootFactTableName)) { + throw new IllegalArgumentException("Computed column has to be defined on fact table"); + } + + if ("true".equals(System.getProperty("needCheckCC"))) { //conditional execute this because of the calcite dependency is to available every where + try { + CalciteParser.ensureAliasInExpr(expression, aliasSet); + } catch (Exception e) { + String legacyHandled = handleLegacyCC(expression, rootFactTableName, aliasSet); + if (legacyHandled != null) { + expression = legacyHandled; + } else { + throw e; + } + } + } + } + + private String handleLegacyCC(String expr, String rootFact, Set<String> aliasSet) { + try { + CalciteParser.ensureNoAliasInExpr(expr); + String ret = CalciteParser.insertAliasInExpr(expr, rootFact); + CalciteParser.ensureAliasInExpr(ret, aliasSet); + return ret; + } catch (Exception e) { + logger.error("failed to handle legacy CC " + expr); + return null; + } } public String getFullName() { @@ -63,6 +108,10 @@ public class ComputedColumnDesc implements Serializable { return tableIdentity; } + public String getTableAlias() { + return tableAlias; + } + public String getColumnName() { return columnName; } @@ -90,6 +139,8 @@ public class ComputedColumnDesc implements Serializable { if (!tableIdentity.equals(that.tableIdentity)) return false; + if (!StringUtils.equals(tableAlias, that.tableAlias)) + return false; if (!columnName.equals(that.columnName)) return false; if (!expression.equals(that.expression)) @@ -100,6 +151,8 @@ public class ComputedColumnDesc implements Serializable { @Override public int hashCode() { int result = tableIdentity.hashCode(); + if (tableAlias != null) + result = 31 * result + tableAlias.hashCode(); result = 31 * result + columnName.hashCode(); result = 31 * result + expression.hashCode(); result = 31 * result + datatype.hashCode(); http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java index 2037420..8fcbc4c 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java @@ -22,6 +22,7 @@ import java.io.Serializable; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -189,6 +190,10 @@ public class DataModelDesc extends RootPersistentEntity { return factTableRefs; } + public Map<String, TableRef> getAliasMap() { + return Collections.unmodifiableMap(aliasMap); + } + public Set<TableRef> getLookupTables() { return lookupTableRefs; } @@ -430,8 +435,9 @@ public class DataModelDesc extends RootPersistentEntity { String alias = join.getAlias(); if (alias == null) { alias = tableDesc.getName(); - join.setAlias(alias); } + alias = alias.toUpperCase(); + join.setAlias(alias); TableRef ref = new TableRef(this, alias, tableDesc); @@ -494,25 +500,26 @@ public class DataModelDesc extends RootPersistentEntity { for (ComputedColumnDesc newCC : this.computedColumnDescs) { - newCC.init(); - final String newCCName = newCC.getFullName(); + newCC.init(aliasMap.keySet(), rootFactTableRef.getAlias()); + final String newCCFullName = newCC.getFullName(); final String newCCColumnName = newCC.getColumnName(); for (Pair<ComputedColumnDesc, DataModelDesc> pair : existingCCs) { DataModelDesc dataModelDesc = pair.getSecond(); ComputedColumnDesc cc = pair.getFirst(); - if (StringUtils.equalsIgnoreCase(cc.getFullName(), newCCName) && !(cc.equals(newCC))) { + if (StringUtils.equalsIgnoreCase(cc.getFullName(), newCCFullName) && !(cc.equals(newCC))) { throw new IllegalArgumentException(String.format( "Column name for computed column %s is already used in model %s, you should apply the same expression ' %s ' here, or use a different column name.", - newCCName, dataModelDesc.getName(), cc.getExpression())); + newCCFullName, dataModelDesc.getName(), cc.getExpression())); } if (isTwoCCDefinitionEquals(cc.getExpression(), newCC.getExpression()) && !StringUtils.equalsIgnoreCase(cc.getColumnName(), newCCColumnName)) { throw new IllegalArgumentException(String.format( - "Expression for computed column %s is already used in model %s, you should use the same column name with ' %s ' .", - newCCName, dataModelDesc.getName(), cc.getColumnName())); + "Expression %s in computed column %s is already defined by computed column %s from model %s, you should use the same column name: ' %s ' .", + newCC.getExpression(), newCCFullName, cc.getFullName(), dataModelDesc.getName(), + cc.getColumnName())); } } existingCCs.add(Pair.newPair(newCC, this)); http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java index 01f0b19..08cbf09 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java @@ -21,15 +21,11 @@ package org.apache.kylin.metadata.model.tool; import java.util.Collections; import java.util.Comparator; import java.util.List; -import java.util.Objects; import java.util.Set; -import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlIdentifier; -import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; -import org.apache.calcite.sql.SqlOrderBy; import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; @@ -54,8 +50,8 @@ public class CalciteParser { try { selectList = ((SqlSelect) CalciteParser.parse(sql)).getSelectList(); } catch (SqlParseException e) { - throw new RuntimeException("Failed to parse expression \'" + sql - + "\', please make sure the expression is valid and no table name exists in the expression"); + throw new RuntimeException( + "Failed to parse expression \'" + sql + "\', please make sure the expression is valid"); } Preconditions.checkArgument(selectList.size() == 1, @@ -66,100 +62,34 @@ public class CalciteParser { public static SqlNode getExpNode(String expr) { return getOnlySelectNode("select " + expr + " from t"); - } - public static SqlNode getFromNode(String sql) { - SqlNode node = null; - try { - node = CalciteParser.parse(sql); - } catch (SqlParseException e) { - throw new RuntimeException( - "Failed to parse expression \'" + sql + "\', please make sure the expression is valid"); - } - //When the sql have limit clause, calcite will parse it as a SqlOrder Object. - SqlNode fromNode = null; - if (node instanceof SqlOrderBy) { - SqlOrderBy orderBy = (SqlOrderBy) node; - fromNode = ((SqlSelect) orderBy.query).getFrom(); - } else { - SqlSelect sqlSelect = (SqlSelect) node; - fromNode = sqlSelect.getFrom(); - } - return fromNode; - } - - public static boolean isNodeEqual(SqlNode node0, SqlNode node1) { - if (node0 == null) { - return node1 == null; - } else if (node1 == null) { - return false; - } - - if (!Objects.equals(node0.getClass().getSimpleName(), node1.getClass().getSimpleName())) { - return false; - } + public static void ensureAliasInExpr(final String expr, final Set<String> aliasSet) { + SqlNode sqlNode = getExpNode(expr); - if (node0 instanceof SqlCall) { - SqlCall thisNode = (SqlCall) node0; - SqlCall thatNode = (SqlCall) node1; - if (!thisNode.getOperator().getName().equalsIgnoreCase(thatNode.getOperator().getName())) { - return false; - } - return isNodeEqual(thisNode.getOperandList(), thatNode.getOperandList()); - } - if (node0 instanceof SqlLiteral) { - SqlLiteral thisNode = (SqlLiteral) node0; - SqlLiteral thatNode = (SqlLiteral) node1; - return Objects.equals(thisNode.getValue(), thatNode.getValue()); - } - if (node0 instanceof SqlNodeList) { - SqlNodeList thisNode = (SqlNodeList) node0; - SqlNodeList thatNode = (SqlNodeList) node1; - if (thisNode.getList().size() != thatNode.getList().size()) { - return false; - } - for (int i = 0; i < thisNode.getList().size(); i++) { - SqlNode thisChild = thisNode.getList().get(i); - final SqlNode thatChild = thatNode.getList().get(i); - if (!isNodeEqual(thisChild, thatChild)) { - return false; + SqlVisitor sqlVisitor = new SqlBasicVisitor() { + @Override + public Object visit(SqlIdentifier id) { + if (id.names.size() < 2 || !aliasSet.contains(id.names.get(0))) { + throw new IllegalArgumentException("Column Identifier in the computed column " + expr + + "expression should comply to ALIAS.COLUMN "); } + return null; } - return true; - } - if (node0 instanceof SqlIdentifier) { - SqlIdentifier thisNode = (SqlIdentifier) node0; - SqlIdentifier thatNode = (SqlIdentifier) node1; - // compare ignore table alias.eg: expression like "a.b + a.c + a.d" ,alias a will be ignored when compared - String name0 = thisNode.names.get(thisNode.names.size() - 1).replace("\"", ""); - String name1 = thatNode.names.get(thatNode.names.size() - 1).replace("\"", ""); - return name0.equalsIgnoreCase(name1); - } - - return false; - } + }; - private static boolean isNodeEqual(List<SqlNode> operands0, List<SqlNode> operands1) { - if (operands0.size() != operands1.size()) { - return false; - } - for (int i = 0; i < operands0.size(); i++) { - if (!isNodeEqual(operands0.get(i), operands1.get(i))) { - return false; - } - } - return true; + sqlNode.accept(sqlVisitor); } - public static void ensureNoTableNameExists(String expr) { + public static void ensureNoAliasInExpr(String expr) { SqlNode sqlNode = getExpNode(expr); SqlVisitor sqlVisitor = new SqlBasicVisitor() { @Override public Object visit(SqlIdentifier id) { if (id.names.size() > 1) { - throw new IllegalArgumentException("SqlIdentifier " + id + " contains DB/Table name"); + throw new IllegalArgumentException( + "Column Identifier in the computed column expression should only contain COLUMN"); } return null; } http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java index 82180a9..df5450c 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java @@ -405,7 +405,7 @@ public class ProjectManager { projectInstance.removeExtFilter(filterName); updateProject(projectInstance); } - + public ProjectInstance getProjectOfModel(String model) { for (ProjectInstance prj : projectMap.values()) { if (prj.getModels().contains(model)) @@ -413,7 +413,7 @@ public class ProjectManager { } throw new IllegalStateException("No project found for model " + model); } - + public List<ProjectInstance> findProjects(RealizationType type, String realizationName) { List<ProjectInstance> result = Lists.newArrayList(); for (ProjectInstance prj : projectMap.values()) { @@ -465,11 +465,10 @@ public class ProjectManager { l2Cache.listExposedTables(norm(project)); } - public List<ColumnDesc> listExposedColumns(String project, TableDesc tableDesc) { + public List<ColumnDesc> listExposedColumns(String project, TableDesc tableDesc, boolean exposeMore) { Set<ColumnDesc> exposedColumns = l2Cache.listExposedColumns(norm(project), tableDesc.getIdentity()); - if (config.isPushDownEnabled()) { - // take care of computed columns + if (exposeMore) { Set<ColumnDesc> dedup = Sets.newHashSet(tableDesc.getColumns()); dedup.addAll(exposedColumns); return Lists.newArrayList(dedup); http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java b/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java index 2da6c9e..6483590 100644 --- a/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java @@ -103,22 +103,6 @@ public class CalciteParserTest { } - @Test - public void testEqual() throws SqlParseException { - String sql0 = "select a.a + a.b + a.c from t as a"; - String sql1 = "select (((a . a + a.b + a.c))) from t as a"; - String sql2 = "select (a + b) + c from t"; - String sql3 = "select a.a + (a.b + a.c) from t as a"; - - SqlNode sn0 = CalciteParser.getOnlySelectNode(sql0); - SqlNode sn1 = CalciteParser.getOnlySelectNode(sql1); - SqlNode sn2 = CalciteParser.getOnlySelectNode(sql2); - SqlNode sn3 = CalciteParser.getOnlySelectNode(sql3); - - assertEquals(true, CalciteParser.isNodeEqual(sn0, sn1)); - assertEquals(true, CalciteParser.isNodeEqual(sn0, sn2)); - assertEquals(false, CalciteParser.isNodeEqual(sn0, sn3)); - } @Test public void testPosWithBrackets() throws SqlParseException { http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java index ea3811c..b40aa5d 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java @@ -83,9 +83,10 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { if (cubeDesc.getConfig().isSkippingEmptySegments() && cubeSeg.getInputRecords() == 0) { logger.info("Skip cube segment {} because its input record is 0", cubeSeg); continue; - } + } - scanner = new CubeSegmentScanner(cubeSeg, request.getCuboid(), request.getDimensions(), request.getGroups(), request.getMetrics(), request.getFilter(), request.getHavingFilter(), request.getContext()); + scanner = new CubeSegmentScanner(cubeSeg, request.getCuboid(), request.getDimensions(), request.getGroups(), + request.getMetrics(), request.getFilter(), request.getHavingFilter(), request.getContext()); if (!scanner.isSegmentSkipped()) scanners.add(scanner); } @@ -97,7 +98,8 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { request.getMetrics(), returnTupleInfo, request.getContext(), sqlDigest); } - protected GTCubeStorageQueryRequest getStorageQueryRequest(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) { + protected GTCubeStorageQueryRequest getStorageQueryRequest(StorageContext context, SQLDigest sqlDigest, + TupleInfo returnTupleInfo) { context.setStorageQuery(this); //cope with queries with no aggregations @@ -143,18 +145,23 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { TupleFilter.collectColumns(filterD, filterColumnD); // set limit push down - enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, groupsD, filterD, loosenedColumnD, sqlDigest.aggregations, context); + enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, groupsD, filterD, loosenedColumnD, + sqlDigest.aggregations, context); // set whether to aggregate results from multiple partitions enableStreamAggregateIfBeneficial(cuboid, groupsD, context); // set query deadline context.setDeadline(cubeInstance); // push down having clause filter if possible - TupleFilter havingFilter = checkHavingCanPushDown(sqlDigest.havingFilter, groupsD, sqlDigest.aggregations, metrics); + TupleFilter havingFilter = checkHavingCanPushDown(sqlDigest.havingFilter, groupsD, sqlDigest.aggregations, + metrics); - logger.info("Cuboid identified: cube={}, cuboidId={}, groupsD={}, filterD={}, limitPushdown={}, storageAggr={}", cubeInstance.getName(), cuboid.getId(), groupsD, filterColumnD, context.getFinalPushDownLimit(), context.isNeedStorageAggregation()); + logger.info("Cuboid identified: cube={}, cuboidId={}, groupsD={}, filterD={}, limitPushdown={}, storageAggr={}", + cubeInstance.getName(), cuboid.getId(), groupsD, filterColumnD, context.getFinalPushDownLimit(), + context.isNeedStorageAggregation()); - return new GTCubeStorageQueryRequest(cuboid, dimensionsD, groupsD, filterColumnD, metrics, filterD, havingFilter, context); + return new GTCubeStorageQueryRequest(cuboid, dimensionsD, groupsD, filterColumnD, metrics, filterD, + havingFilter, context); } protected abstract String getGTStorage(); @@ -163,11 +170,13 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { return Cuboid.identifyCuboid(cubeDesc, dimensionsD, metrics); } - protected ITupleConverter newCubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, int[] gtColIdx, TupleInfo tupleInfo) { + protected ITupleConverter newCubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, + Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, int[] gtColIdx, TupleInfo tupleInfo) { return new CubeTupleConverter(cubeSeg, cuboid, selectedDimensions, selectedMetrics, gtColIdx, tupleInfo); } - protected void buildDimensionsAndMetrics(SQLDigest sqlDigest, Collection<TblColRef> dimensions, Collection<FunctionDesc> metrics) { + protected void buildDimensionsAndMetrics(SQLDigest sqlDigest, Collection<TblColRef> dimensions, + Collection<FunctionDesc> metrics) { for (FunctionDesc func : sqlDigest.aggregations) { if (!func.isDimensionAsMetric()) { // use the FunctionDesc from cube desc as much as possible, that has more info such as HLLC precision @@ -177,7 +186,8 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { for (TblColRef column : sqlDigest.allColumns) { // skip measure columns - if (sqlDigest.metricColumns.contains(column) && !(sqlDigest.groupbyColumns.contains(column) || sqlDigest.filterColumns.contains(column))) { + if (sqlDigest.metricColumns.contains(column) + && !(sqlDigest.groupbyColumns.contains(column) || sqlDigest.filterColumns.contains(column))) { continue; } @@ -226,7 +236,8 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { if (f instanceof CompareTupleFilter) { CompareTupleFilter compFilter = (CompareTupleFilter) f; // is COL=const ? - if (compFilter.getOperator() == FilterOperatorEnum.EQ && compFilter.getValues().size() == 1 && compFilter.getColumn() != null) { + if (compFilter.getOperator() == FilterOperatorEnum.EQ && compFilter.getValues().size() == 1 + && compFilter.getColumn() != null) { result.add(compFilter.getColumn()); } } @@ -253,7 +264,8 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { return resultD; } - public boolean isNeedStorageAggregation(Cuboid cuboid, Collection<TblColRef> groupD, Collection<TblColRef> singleValueD) { + public boolean isNeedStorageAggregation(Cuboid cuboid, Collection<TblColRef> groupD, + Collection<TblColRef> singleValueD) { HashSet<TblColRef> temp = Sets.newHashSet(); temp.addAll(groupD); temp.addAll(singleValueD); @@ -345,7 +357,9 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { } } - private void enableStorageLimitIfPossible(Cuboid cuboid, Collection<TblColRef> groups, Set<TblColRef> derivedPostAggregation, Collection<TblColRef> groupsD, TupleFilter filter, Set<TblColRef> loosenedColumnD, Collection<FunctionDesc> functionDescs, StorageContext context) { + private void enableStorageLimitIfPossible(Cuboid cuboid, Collection<TblColRef> groups, + Set<TblColRef> derivedPostAggregation, Collection<TblColRef> groupsD, TupleFilter filter, + Set<TblColRef> loosenedColumnD, Collection<FunctionDesc> functionDescs, StorageContext context) { boolean possible = true; if (!TupleFilter.isEvaluableRecursively(filter)) { @@ -366,15 +380,17 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { // derived aggregation is bad, unless expanded columns are already in group by if (!groups.containsAll(derivedPostAggregation)) { possible = false; - logger.debug("Storage limit push down is impossible because derived column require post aggregation: " + derivedPostAggregation); + logger.debug("Storage limit push down is impossible because derived column require post aggregation: " + + derivedPostAggregation); } //if groupsD is clustered at "head" of the rowkey, then limit push down is possible int size = groupsD.size(); if (!groupsD.containsAll(cuboid.getColumns().subList(0, size))) { possible = false; - logger.debug("Storage limit push down is impossible because groupD is not clustered at head, groupsD: " + groupsD // - + " with cuboid columns: " + cuboid.getColumns()); + logger.debug( + "Storage limit push down is impossible because groupD is not clustered at head, groupsD: " + groupsD // + + " with cuboid columns: " + cuboid.getColumns()); } //if exists measures like max(cal_dt), then it's not a perfect cuboid match, cannot apply limit @@ -402,7 +418,8 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { } if (!shardByInGroups.isEmpty()) { enabled = false; - logger.debug("Aggregate partition results is not beneficial because shard by columns in groupD: " + shardByInGroups); + logger.debug("Aggregate partition results is not beneficial because shard by columns in groupD: " + + shardByInGroups); } if (!context.isNeedStorageAggregation()) { @@ -434,7 +451,8 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { } } - private TupleFilter checkHavingCanPushDown(TupleFilter havingFilter, Set<TblColRef> groupsD, List<FunctionDesc> aggregations, Set<FunctionDesc> metrics) { + private TupleFilter checkHavingCanPushDown(TupleFilter havingFilter, Set<TblColRef> groupsD, + List<FunctionDesc> aggregations, Set<FunctionDesc> metrics) { // must have only one segment Segments<CubeSegment> readySegs = cubeInstance.getSegments(SegmentStatusEnum.READY); if (readySegs.size() != 1) http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json index 27acdd3..efed0f9 100644 --- a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json +++ b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json @@ -131,13 +131,23 @@ }, { "name": "SELLER_COUNTRY_ABBR", - "table": "SELLER_ACCOUNT", - "column": "COUNTRY_ABBR" + "table": "TEST_KYLIN_FACT", + "column": "SELLER_COUNTRY_ABBR" }, { "name": "BUYER_COUNTRY_ABBR", - "table": "BUYER_ACCOUNT", - "column": "COUNTRY_ABBR" + "table": "TEST_KYLIN_FACT", + "column": "BUYER_COUNTRY_ABBR" + }, + { + "name": "SELLER_ID_AND_COUNTRY_NAME", + "table": "TEST_KYLIN_FACT", + "column": "SELLER_ID_AND_COUNTRY_NAME" + }, + { + "name": "BUYER_ID_AND_COUNTRY_NAME", + "table": "TEST_KYLIN_FACT", + "column": "BUYER_ID_AND_COUNTRY_NAME" }, { "name": "BUYER_COUNTRY_NAME", @@ -443,11 +453,19 @@ "encoding": "dict" }, { - "column": "SELLER_ACCOUNT.COUNTRY_ABBR", + "column": "TEST_KYLIN_FACT.SELLER_COUNTRY_ABBR", + "encoding": "dict" + }, + { + "column": "TEST_KYLIN_FACT.BUYER_COUNTRY_ABBR", + "encoding": "dict" + }, + { + "column": "TEST_KYLIN_FACT.SELLER_ID_AND_COUNTRY_NAME", "encoding": "dict" }, { - "column": "BUYER_ACCOUNT.COUNTRY_ABBR", + "column": "TEST_KYLIN_FACT.BUYER_ID_AND_COUNTRY_NAME", "encoding": "dict" } ] @@ -536,7 +554,7 @@ "TEST_KYLIN_FACT.DEAL_YEAR" ] ], - "dim_cap" : 3 + "dim_cap": 3 } }, { @@ -562,8 +580,10 @@ "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", - "SELLER_ACCOUNT.COUNTRY_ABBR", - "BUYER_ACCOUNT.COUNTRY_ABBR" + "TEST_KYLIN_FACT.SELLER_COUNTRY_ABBR", + "TEST_KYLIN_FACT.BUYER_COUNTRY_ABBR", + "TEST_KYLIN_FACT.SELLER_ID_AND_COUNTRY_NAME", + "TEST_KYLIN_FACT.BUYER_ID_AND_COUNTRY_NAME" ], "select_rule": { "hierarchy_dims": [], @@ -581,8 +601,10 @@ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", "TEST_KYLIN_FACT.LSTG_SITE_ID", "TEST_KYLIN_FACT.SLR_SEGMENT_CD", - "SELLER_ACCOUNT.COUNTRY_ABBR", - "BUYER_ACCOUNT.COUNTRY_ABBR" + "TEST_KYLIN_FACT.SELLER_COUNTRY_ABBR", + "TEST_KYLIN_FACT.BUYER_COUNTRY_ABBR", + "TEST_KYLIN_FACT.SELLER_ID_AND_COUNTRY_NAME", + "TEST_KYLIN_FACT.BUYER_ID_AND_COUNTRY_NAME" ], [ "TEST_KYLIN_FACT.SELLER_ID", @@ -602,7 +624,7 @@ "BUYER_COUNTRY.NAME" ] ], - "dim_cap" : 5 + "dim_cap": 5 } } ], http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json index f7ebfc8..6cb0722 100644 --- a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json +++ b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json @@ -131,13 +131,23 @@ }, { "name": "SELLER_COUNTRY_ABBR", - "table": "SELLER_ACCOUNT", - "column": "COUNTRY_ABBR" + "table": "TEST_KYLIN_FACT", + "column": "SELLER_COUNTRY_ABBR" }, { "name": "BUYER_COUNTRY_ABBR", - "table": "BUYER_ACCOUNT", - "column": "COUNTRY_ABBR" + "table": "TEST_KYLIN_FACT", + "column": "BUYER_COUNTRY_ABBR" + }, + { + "name": "SELLER_ID_AND_COUNTRY_NAME", + "table": "TEST_KYLIN_FACT", + "column": "SELLER_ID_AND_COUNTRY_NAME" + }, + { + "name": "BUYER_ID_AND_COUNTRY_NAME", + "table": "TEST_KYLIN_FACT", + "column": "BUYER_ID_AND_COUNTRY_NAME" }, { "name": "BUYER_COUNTRY_NAME", @@ -454,11 +464,19 @@ "encoding": "dict" }, { - "column": "SELLER_ACCOUNT.COUNTRY_ABBR", + "column": "TEST_KYLIN_FACT.SELLER_COUNTRY_ABBR", + "encoding": "dict" + }, + { + "column": "TEST_KYLIN_FACT.BUYER_COUNTRY_ABBR", + "encoding": "dict" + }, + { + "column": "TEST_KYLIN_FACT.SELLER_ID_AND_COUNTRY_NAME", "encoding": "dict" }, { - "column": "BUYER_ACCOUNT.COUNTRY_ABBR", + "column": "TEST_KYLIN_FACT.BUYER_ID_AND_COUNTRY_NAME", "encoding": "dict" } ] @@ -548,7 +566,7 @@ "TEST_KYLIN_FACT.DEAL_YEAR" ] ], - "dim_cap" : 5 + "dim_cap": 5 } }, { @@ -574,8 +592,10 @@ "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", - "SELLER_ACCOUNT.COUNTRY_ABBR", - "BUYER_ACCOUNT.COUNTRY_ABBR" + "TEST_KYLIN_FACT.SELLER_COUNTRY_ABBR", + "TEST_KYLIN_FACT.BUYER_COUNTRY_ABBR", + "TEST_KYLIN_FACT.SELLER_ID_AND_COUNTRY_NAME", + "TEST_KYLIN_FACT.BUYER_ID_AND_COUNTRY_NAME" ], "select_rule": { "hierarchy_dims": [], @@ -593,8 +613,10 @@ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", "TEST_KYLIN_FACT.LSTG_SITE_ID", "TEST_KYLIN_FACT.SLR_SEGMENT_CD", - "SELLER_ACCOUNT.COUNTRY_ABBR", - "BUYER_ACCOUNT.COUNTRY_ABBR" + "TEST_KYLIN_FACT.SELLER_COUNTRY_ABBR", + "TEST_KYLIN_FACT.BUYER_COUNTRY_ABBR", + "TEST_KYLIN_FACT.SELLER_ID_AND_COUNTRY_NAME", + "TEST_KYLIN_FACT.BUYER_ID_AND_COUNTRY_NAME" ], [ "TEST_KYLIN_FACT.SELLER_ID", @@ -614,7 +636,7 @@ "BUYER_COUNTRY.NAME" ] ], - "dim_cap" : 3 + "dim_cap": 3 } } ], http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/examples/test_case_data/localmeta/cube_desc/ut_inner_join_cube_partial.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/ut_inner_join_cube_partial.json b/examples/test_case_data/localmeta/cube_desc/ut_inner_join_cube_partial.json index 11d65bd..0e64166 100644 --- a/examples/test_case_data/localmeta/cube_desc/ut_inner_join_cube_partial.json +++ b/examples/test_case_data/localmeta/cube_desc/ut_inner_join_cube_partial.json @@ -130,13 +130,13 @@ }, { "name": "SELLER_COUNTRY_ABBR", - "table": "SELLER_ACCOUNT", - "column": "COUNTRY_ABBR" + "table": "TEST_KYLIN_FACT", + "column": "SELLER_COUNTRY_ABBR" }, { "name": "BUYER_COUNTRY_ABBR", - "table": "BUYER_ACCOUNT", - "column": "COUNTRY_ABBR" + "table": "TEST_KYLIN_FACT", + "column": "BUYER_COUNTRY_ABBR" }, { "name": "BUYER_COUNTRY_NAME", @@ -237,18 +237,20 @@ "encoding": "dict" }, { - "column": "SELLER_ACCOUNT.COUNTRY_ABBR", + "column": "TEST_KYLIN_FACT.SELLER_COUNTRY_ABBR", "encoding": "dict" }, { - "column": "BUYER_ACCOUNT.COUNTRY_ABBR", + "column": "TEST_KYLIN_FACT.BUYER_COUNTRY_ABBR", "encoding": "dict" } ] }, "signature": null, "null_string": null, - "hbase_mapping": { "column_family": [ ] }, + "hbase_mapping": { + "column_family": [] + }, "aggregation_groups": [ { "includes": [ @@ -280,7 +282,7 @@ "TEST_KYLIN_FACT.DEAL_YEAR" ] ], - "dim_cap" : 3 + "dim_cap": 3 } }, { @@ -306,8 +308,8 @@ "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", - "SELLER_ACCOUNT.COUNTRY_ABBR", - "BUYER_ACCOUNT.COUNTRY_ABBR" + "TEST_KYLIN_FACT.SELLER_COUNTRY_ABBR", + "TEST_KYLIN_FACT.BUYER_COUNTRY_ABBR" ], "select_rule": { "hierarchy_dims": [], @@ -325,8 +327,8 @@ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", "TEST_KYLIN_FACT.LSTG_SITE_ID", "TEST_KYLIN_FACT.SLR_SEGMENT_CD", - "SELLER_ACCOUNT.COUNTRY_ABBR", - "BUYER_ACCOUNT.COUNTRY_ABBR" + "TEST_KYLIN_FACT.SELLER_COUNTRY_ABBR", + "TEST_KYLIN_FACT.BUYER_COUNTRY_ABBR" ], [ "TEST_KYLIN_FACT.SELLER_ID", @@ -346,7 +348,7 @@ "BUYER_COUNTRY.NAME" ] ], - "dim_cap" : 3 + "dim_cap": 3 } } ], http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json b/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json index 5225cf3..dbedc66 100644 --- a/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json +++ b/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json @@ -127,21 +127,47 @@ "columnName": "DEAL_AMOUNT", "expression": "PRICE * ITEM_COUNT", "datatype": "decimal", - "comment": "deal amount of inner join model" + "comment": "deal amount of inner join model (with legacy expression format)" }, { "tableIdentity": "DEFAULT.TEST_KYLIN_FACT", + "tableAlias": "TEST_KYLIN_FACT", "columnName": "DEAL_YEAR", - "expression": "year(CAL_DT)", + "expression": "year(TEST_KYLIN_FACT.CAL_DT)", "datatype": "integer", - "comment": "the year of the deal of the inner join model" + "comment": "the year of the deal" }, { - "tableIdentity": "DEFAULT.TEST_ACCOUNT", - "columnName": "COUNTRY_ABBR", - "expression": "SUBSTR(ACCOUNT_COUNTRY,0,1)", + "tableIdentity": "DEFAULT.TEST_KYLIN_FACT", + "tableAlias": "TEST_KYLIN_FACT", + "columnName": "BUYER_ID_AND_COUNTRY_NAME", + "expression": "CONCAT(BUYER_ACCOUNT.ACCOUNT_ID, BUYER_COUNTRY.NAME)", + "datatype": "string", + "comment": "synthetically concat buyer's account id and buyer country" + }, + { + "tableIdentity": "DEFAULT.TEST_KYLIN_FACT", + "tableAlias": "TEST_KYLIN_FACT", + "columnName": "SELLER_ID_AND_COUNTRY_NAME", + "expression": "CONCAT(SELLER_ACCOUNT.ACCOUNT_ID, SELLER_COUNTRY.NAME)", + "datatype": "string", + "comment": "synthetically concat seller's account id and seller country" + }, + { + "tableIdentity": "DEFAULT.TEST_KYLIN_FACT", + "tableAlias": "TEST_KYLIN_FACT", + "columnName": "BUYER_COUNTRY_ABBR", + "expression": "SUBSTR(BUYER_ACCOUNT.ACCOUNT_COUNTRY,0,1)", + "datatype": "string", + "comment": "first char of country of buyer account" + }, + { + "tableIdentity": "DEFAULT.TEST_KYLIN_FACT", + "tableAlias": "TEST_KYLIN_FACT", + "columnName": "SELLER_COUNTRY_ABBR", + "expression": "SUBSTR(SELLER_ACCOUNT.ACCOUNT_COUNTRY,0,1)", "datatype": "string", - "comment": "first char of country of the inner join model" + "comment": "first char of country of seller account" } ], "dimensions": [ @@ -157,7 +183,11 @@ "SLR_SEGMENT_CD", "SELLER_ID", "TEST_COUNT_DISTINCT_BITMAP", - "DEAL_YEAR" + "DEAL_YEAR", + "SELLER_COUNTRY_ABBR", + "BUYER_COUNTRY_ABBR", + "SELLER_ID_AND_COUNTRY_NAME", + "BUYER_ID_AND_COUNTRY_NAME" ] }, { @@ -177,8 +207,7 @@ "ACCOUNT_BUYER_LEVEL", "ACCOUNT_SELLER_LEVEL", "ACCOUNT_COUNTRY", - "ACCOUNT_CONTACT", - "COUNTRY_ABBR" + "ACCOUNT_CONTACT" ] }, { @@ -188,8 +217,7 @@ "ACCOUNT_BUYER_LEVEL", "ACCOUNT_SELLER_LEVEL", "ACCOUNT_COUNTRY", - "ACCOUNT_CONTACT", - "COUNTRY_ABBR" + "ACCOUNT_CONTACT" ] }, { http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json b/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json index 0058425..7220b4d 100644 --- a/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json +++ b/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json @@ -125,23 +125,49 @@ { "tableIdentity": "DEFAULT.TEST_KYLIN_FACT", "columnName": "DEAL_AMOUNT", - "expression": "PRICE * ITEM_COUNT", + "expression": "PRICE * ITEM_COUNT", "datatype": "decimal", - "comment": "deal amount of left join model" + "comment": "deal amount of left join model (with legacy expression format)" }, { "tableIdentity": "DEFAULT.TEST_KYLIN_FACT", + "tableAlias": "TEST_KYLIN_FACT", "columnName": "DEAL_YEAR", - "expression": "year(CAL_DT)", + "expression": "year(TEST_KYLIN_FACT.CAL_DT)", "datatype": "integer", - "comment": "the year of the deal of the left join model" + "comment": "the year of the deal" }, { - "tableIdentity": "DEFAULT.TEST_ACCOUNT", - "columnName": "COUNTRY_ABBR", - "expression": "SUBSTR(ACCOUNT_COUNTRY,0,1)", + "tableIdentity": "DEFAULT.TEST_KYLIN_FACT", + "tableAlias": "TEST_KYLIN_FACT", + "columnName": "BUYER_ID_AND_COUNTRY_NAME", + "expression": "CONCAT(BUYER_ACCOUNT.ACCOUNT_ID, BUYER_COUNTRY.NAME)", + "datatype": "string", + "comment": "synthetically concat buyer's account id and buyer country" + }, + { + "tableIdentity": "DEFAULT.TEST_KYLIN_FACT", + "tableAlias": "TEST_KYLIN_FACT", + "columnName": "SELLER_ID_AND_COUNTRY_NAME", + "expression": "CONCAT(SELLER_ACCOUNT.ACCOUNT_ID, SELLER_COUNTRY.NAME)", + "datatype": "string", + "comment": "synthetically concat seller's account id and seller country" + }, + { + "tableIdentity": "DEFAULT.TEST_KYLIN_FACT", + "tableAlias": "TEST_KYLIN_FACT", + "columnName": "BUYER_COUNTRY_ABBR", + "expression": "SUBSTR(BUYER_ACCOUNT.ACCOUNT_COUNTRY,0,1)", + "datatype": "string", + "comment": "first char of country of buyer account" + }, + { + "tableIdentity": "DEFAULT.TEST_KYLIN_FACT", + "tableAlias": "TEST_KYLIN_FACT", + "columnName": "SELLER_COUNTRY_ABBR", + "expression": "SUBSTR(SELLER_ACCOUNT.ACCOUNT_COUNTRY,0,1)", "datatype": "string", - "comment": "first char of country of the left join model" + "comment": "first char of country of seller account" } ], "dimensions": [ @@ -157,7 +183,11 @@ "SLR_SEGMENT_CD", "SELLER_ID", "TEST_COUNT_DISTINCT_BITMAP", - "DEAL_YEAR" + "DEAL_YEAR", + "SELLER_COUNTRY_ABBR", + "BUYER_COUNTRY_ABBR", + "SELLER_ID_AND_COUNTRY_NAME", + "BUYER_ID_AND_COUNTRY_NAME" ] }, { @@ -177,8 +207,7 @@ "ACCOUNT_BUYER_LEVEL", "ACCOUNT_SELLER_LEVEL", "ACCOUNT_COUNTRY", - "ACCOUNT_CONTACT", - "COUNTRY_ABBR" + "ACCOUNT_CONTACT" ] }, { @@ -188,8 +217,7 @@ "ACCOUNT_BUYER_LEVEL", "ACCOUNT_SELLER_LEVEL", "ACCOUNT_COUNTRY", - "ACCOUNT_CONTACT", - "COUNTRY_ABBR" + "ACCOUNT_CONTACT" ] }, { http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/kylin-it/src/test/resources/query/sql_computedcolumn/query01.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_computedcolumn/query01.sql b/kylin-it/src/test/resources/query/sql_computedcolumn/query01.sql index 5661c79..6aba6ce 100644 --- a/kylin-it/src/test/resources/query/sql_computedcolumn/query01.sql +++ b/kylin-it/src/test/resources/query/sql_computedcolumn/query01.sql @@ -10,5 +10,5 @@ ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TES INNER JOIN TEST_COUNTRY as SELLER_COUNTRY ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY -where SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL=1 and SELLER_ACCOUNT.COUNTRY_ABBR in ('I', 'F') +where SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL=1 and TEST_KYLIN_FACT.SELLER_COUNTRY_ABBR in ('I', 'F') group by SELLER_COUNTRY.NAME, DEAL_YEAR http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/kylin-it/src/test/resources/query/sql_computedcolumn/query02.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_computedcolumn/query02.sql b/kylin-it/src/test/resources/query/sql_computedcolumn/query02.sql index 0e84301..3feafee 100644 --- a/kylin-it/src/test/resources/query/sql_computedcolumn/query02.sql +++ b/kylin-it/src/test/resources/query/sql_computedcolumn/query02.sql @@ -17,5 +17,5 @@ INNER JOIN TEST_COUNTRY as SELLER_COUNTRY ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY -where SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL=1 and SELLER_ACCOUNT.COUNTRY_ABBR in ('T', 'R') and BUYER_ACCOUNT.COUNTRY_ABBR in ('T', 'R') +where SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL=1 and TEST_KYLIN_FACT.SELLER_COUNTRY_ABBR in ('T', 'R') and TEST_KYLIN_FACT.BUYER_COUNTRY_ABBR in ('T', 'R') group by SELLER_COUNTRY.NAME, DEAL_YEAR http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql b/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql index ff8a570..06eabeb 100644 --- a/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql +++ b/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql @@ -17,5 +17,5 @@ INNER JOIN TEST_COUNTRY as SELLER_COUNTRY ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY -where SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL=1 and "SELLER_ACCOUNT"."COUNTRY_ABBR" in ('T', 'R') and BUYER_ACCOUNT.COUNTRY_ABBR in ('T', 'R') +where SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL=1 and "TEST_KYLIN_FACT"."SELLER_COUNTRY_ABBR" in ('T', 'R') and TEST_KYLIN_FACT.BUYER_COUNTRY_ABBR in ('T', 'R') group by SELLER_COUNTRY.NAME, DEAL_YEAR http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/kylin-it/src/test/resources/query/sql_verifyCount/query01.sql.expected ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_verifyCount/query01.sql.expected b/kylin-it/src/test/resources/query/sql_verifyCount/query01.sql.expected index 11636e7..d2f0772 100644 --- a/kylin-it/src/test/resources/query/sql_verifyCount/query01.sql.expected +++ b/kylin-it/src/test/resources/query/sql_verifyCount/query01.sql.expected @@ -1,2 +1,2 @@ * -13 +17 http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/kylin-it/src/test/resources/query/sql_verifyCount/query03.sql.expected ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_verifyCount/query03.sql.expected b/kylin-it/src/test/resources/query/sql_verifyCount/query03.sql.expected index 349103a..3097aec 100644 --- a/kylin-it/src/test/resources/query/sql_verifyCount/query03.sql.expected +++ b/kylin-it/src/test/resources/query/sql_verifyCount/query03.sql.expected @@ -1,2 +1,2 @@ 10 -15 +19 http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/kylin-it/src/test/resources/query/sql_verifyCount/query04.sql.expected ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_verifyCount/query04.sql.expected b/kylin-it/src/test/resources/query/sql_verifyCount/query04.sql.expected index d10e701..0b28614 100644 --- a/kylin-it/src/test/resources/query/sql_verifyCount/query04.sql.expected +++ b/kylin-it/src/test/resources/query/sql_verifyCount/query04.sql.expected @@ -1,2 +1,2 @@ 100 -13 +17 http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/kylin-it/src/test/resources/query/sql_verifyCount/query10.sql.expected ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_verifyCount/query10.sql.expected b/kylin-it/src/test/resources/query/sql_verifyCount/query10.sql.expected index 11636e7..d2f0772 100644 --- a/kylin-it/src/test/resources/query/sql_verifyCount/query10.sql.expected +++ b/kylin-it/src/test/resources/query/sql_verifyCount/query10.sql.expected @@ -1,2 +1,2 @@ * -13 +17 http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/kylin-it/src/test/resources/query/sql_verifyCount/query11.sql.expected ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_verifyCount/query11.sql.expected b/kylin-it/src/test/resources/query/sql_verifyCount/query11.sql.expected index 093e327..a21e30c 100644 --- a/kylin-it/src/test/resources/query/sql_verifyCount/query11.sql.expected +++ b/kylin-it/src/test/resources/query/sql_verifyCount/query11.sql.expected @@ -1,2 +1,2 @@ 10 -13 +17 http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java index 8001880..751e6b0 100644 --- a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java +++ b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java @@ -119,4 +119,4 @@ public class PushDownRunnerJdbcImpl implements IPushDownRunner { } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java index 28ee623..9df5557 100644 --- a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java +++ b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java @@ -71,7 +71,7 @@ public class LookupTableEnumerator implements Enumerator<Object[]> { this.allRows = table.getAllRows(); OLAPTable olapTable = (OLAPTable) olapContext.firstTableScan.getOlapTable(); - this.colDescs = olapTable.getExposedColumns(); + this.colDescs = olapTable.getSourceColumns(); this.current = new Object[colDescs.size()]; reset();
