KYLIN-2575 Experimental feature: Computed Column
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9a812d5f Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9a812d5f Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9a812d5f Branch: refs/heads/master Commit: 9a812d5fc917ddcdb736cd1c91fdfe9c933e603a Parents: d6fdda2 Author: Hongbin Ma <mahong...@apache.org> Authored: Mon May 8 15:49:57 2017 +0800 Committer: liyang-gmt8 <liy...@apache.org> Committed: Mon May 8 16:01:48 2017 +0800 ---------------------------------------------------------------------- .../kylin/cube/CubeCapabilityChecker.java | 11 +- .../java/org/apache/kylin/cube/JoinChecker.java | 60 - .../org/apache/kylin/cube/cuboid/CuboidCLI.java | 7 + .../org/apache/kylin/cube/model/CubeDesc.java | 12 +- .../kylin/cube/cuboid/CuboidSchedulerTest.java | 23 + .../model/validation/rule/FunctionRuleTest.java | 11 +- .../org/apache/kylin/job/JoinedFlatTable.java | 4 +- .../apache/kylin/metadata/MetadataManager.java | 31 +- .../apache/kylin/metadata/model/ColumnDesc.java | 41 +- .../metadata/model/ComputedColumnDesc.java | 95 ++ .../kylin/metadata/model/DataModelDesc.java | 81 +- .../apache/kylin/metadata/model/JoinsTree.java | 2 +- .../kylin/metadata/model/PartitionDesc.java | 10 +- .../apache/kylin/metadata/model/TableDesc.java | 35 +- .../apache/kylin/metadata/model/TableRef.java | 14 +- .../apache/kylin/metadata/model/TblColRef.java | 36 +- .../kylin/metadata/project/ProjectL2Cache.java | 13 +- .../kylin/metadata/model/DataModelDescTest.java | 15 +- .../localmeta/cube_desc/ci_inner_join_cube.json | 1016 ++++++++++------- .../localmeta/cube_desc/ci_left_join_cube.json | 1036 ++++++++++-------- .../model_desc/ci_inner_join_model.json | 37 +- .../model_desc/ci_left_join_model.json | 37 +- .../test_case_data/sandbox/kylin.properties | 2 +- .../kylin/query/CompareQueryBySuffix.java | 37 + .../kylin/query/ICompareQueryTranslator.java | 29 + .../apache/kylin/query/ITKylinQueryTest.java | 5 + .../org/apache/kylin/query/KylinTestBase.java | 25 +- .../query/sql_computedcolumn/query01.sql | 14 + .../sql_computedcolumn/query01.sql.compare | 14 + .../query/sql_computedcolumn/query02.sql | 21 + .../sql_computedcolumn/query02.sql.compare | 21 + .../query/sql_verifyCount/query01.sql.expected | 2 +- .../query/sql_verifyCount/query03.sql.expected | 2 +- .../query/sql_verifyCount/query04.sql.expected | 2 +- .../apache/kylin/query/schema/OLAPSchema.java | 2 +- .../kylin/rest/controller/ModelController.java | 2 +- .../kylin/rest/service/CacheServiceTest.java | 1 - .../kylin/rest/service/ModelServiceTest.java | 97 ++ .../apache/kylin/source/hive/SchemaChecker.java | 2 +- 39 files changed, 1894 insertions(+), 1011 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java index c45144b..20cb0a9 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java @@ -52,12 +52,7 @@ public class CubeCapabilityChecker { CapabilityResult result = new CapabilityResult(); result.capable = false; - // match joins - boolean isJoinMatch = JoinChecker.isJoinMatch(digest.joinDescs, cube); - if (!isJoinMatch) { - logger.info("Exclude cube " + cube.getName() + " because unmatched joins"); - return result; - } + // match joins is ensured at model select // dimensions & measures Collection<TblColRef> dimensionColumns = getDimensionColumns(digest); @@ -106,12 +101,12 @@ public class CubeCapabilityChecker { } if (!unmatchedDimensions.isEmpty()) { - logger.info("Exclude cube " + cube.getName() + " because unmatched dimensions"); + logger.info("Exclude cube " + cube.getName() + " because unmatched dimensions: " + unmatchedDimensions); return result; } if (!unmatchedAggregations.isEmpty()) { - logger.info("Exclude cube " + cube.getName() + " because unmatched aggregations"); + logger.info("Exclude cube " + cube.getName() + " because unmatched aggregations: " + unmatchedAggregations); return result; } http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-cube/src/main/java/org/apache/kylin/cube/JoinChecker.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/JoinChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/JoinChecker.java deleted file mode 100644 index 9068449..0000000 --- a/core-cube/src/main/java/org/apache/kylin/cube/JoinChecker.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.cube; - -import java.util.Collection; - -import org.apache.kylin.metadata.model.JoinDesc; -import org.apache.kylin.metadata.realization.IRealization; - -public class JoinChecker { - - // given ModelChooser has done the model join matching already, this method seems useless - public static boolean isJoinMatch(Collection<JoinDesc> joins, IRealization realization) { -// List<JoinDesc> realizationsJoins = Lists.newArrayList(); -// for (JoinTableDesc joinTable : realization.getModel().getJoinTables()) { -// realizationsJoins.add(joinTable.getJoin()); -// } -// -// for (JoinDesc j : joins) { -// // optiq engine can't decide which one is fk or pk -// String pTable = j.getPrimaryKeyColumns()[0].getTable(); -// String factTable = realization.getModel().getRootFactTable().getTableIdentity(); -// if (factTable.equals(pTable)) { -// j.swapPKFK(); -// } -// -// // check primary key, all PK column should refer to same tale, the Fact Table of cube. -// // Using first column's table name to check. -// String fTable = j.getForeignKeyColumns()[0].getTable(); -// if (!factTable.equals(fTable)) { -// logger.info("Fact Table" + factTable + " not matched in join: " + j + " on cube " + realization.getName()); -// return false; -// } -// -// // The hashcode() function of JoinDesc has been overwritten, -// // which takes into consideration: pk,fk,jointype -// if (!realizationsJoins.contains(j)) { -// logger.info("Query joins don't macth on cube " + realization.getName()); -// return false; -// } -// } - return true; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java index 05efb5e..535f77a 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java @@ -132,10 +132,17 @@ public class CuboidCLI { while (!currentQueue.isEmpty()) { long cuboid = currentQueue.pop(); Collection<Long> spnanningCuboids = scheduler.getSpanningCuboid(cuboid); + nextQueue.addAll(spnanningCuboids); } currentQueue = nextQueue; nextQueue = new LinkedList<Long>(); + + if (i == levels) { + if (!currentQueue.isEmpty()) { + throw new IllegalStateException(); + } + } } return allLevelCounts; http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 53c11a3..dfbe6e7 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -26,6 +26,7 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Arrays; +import java.util.BitSet; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -887,7 +888,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { } } } - + return initDimensionColRef(col); } @@ -939,6 +940,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { for (int i = 0; i < measures.size(); i++) measureIndexLookup.put(measures.get(i).getName(), i); + BitSet checkEachMeasureExist = new BitSet(); for (HBaseColumnFamilyDesc cf : getHbaseMapping().getColumnFamily()) { for (HBaseColumnDesc c : cf.getColumns()) { String[] colMeasureRefs = c.getMeasureRefs(); @@ -946,13 +948,21 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { int[] measureIndex = new int[colMeasureRefs.length]; for (int i = 0; i < colMeasureRefs.length; i++) { measureDescs[i] = measureLookup.get(colMeasureRefs[i]); + checkState(measureDescs[i] != null, "measure desc at (%s) is null", i); measureIndex[i] = measureIndexLookup.get(colMeasureRefs[i]); + checkState(measureIndex[i] >= 0, "measure index at (%s) not positive", i); + + checkEachMeasureExist.set(measureIndex[i]); } c.setMeasures(measureDescs); c.setMeasureIndex(measureIndex); c.setColumnFamilyName(cf.getName()); } } + + for (int i = 0; i < measures.size(); i++) { + checkState(checkEachMeasureExist.get(i), "measure (%s) does not exist in column familyï¼or measure duplicates", measures.get(i)); + } } private void initDictionaryDesc() { http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java index 7db616e..6457770 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java @@ -94,6 +94,10 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase { private CubeDesc getSSBCubeDesc() { return getCubeDescManager().getCubeDesc("ssb"); } + + private CubeDesc getCIInnerJoinCube() { + return getCubeDescManager().getCubeDesc("ci_inner_join_cube"); + } private void testSpanningAndGetParent(CuboidScheduler scheduler, CubeDesc cube, long[] cuboidIds) { for (long cuboidId : cuboidIds) { @@ -219,6 +223,12 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase { CubeDesc cube = getSSBCubeDesc(); CuboidCLI.simulateCuboidGeneration(cube, true); } + + @Test + public void testCuboidGeneration7() { + CubeDesc cube = getCIInnerJoinCube(); + CuboidCLI.simulateCuboidGeneration(cube, true); + } @Test public void testCuboidCounts1() { @@ -286,6 +296,19 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase { } @Test + public void testCuboidCounts6() { + CubeDesc cube = getCIInnerJoinCube(); + CuboidScheduler cuboidScheduler = new CuboidScheduler(cube); + int[] counts = CuboidCLI.calculateAllLevelCount(cube); + printCount(counts); + int sum = 0; + for (Integer x : counts) { + sum += x; + } + assertEquals(cuboidScheduler.getCuboidCount(), sum); + } + + @Test public void testCuboid_onlyBaseCuboid() { for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA, "cube_desc").listFiles()) { if (f.getName().endsWith(".bad")) { http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java index a3456c4..18d84c3 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java @@ -18,7 +18,6 @@ package org.apache.kylin.cube.model.validation.rule; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.io.File; @@ -62,10 +61,8 @@ public class FunctionRuleTest extends LocalFileMetadataTestCase { assertTrue(vContext.getResults().length == 0); } - @Test + @Test(expected = IllegalStateException.class) public void testValidateMeasureNamesDuplicated() throws IOException { - FunctionRule rule = new FunctionRule(); - File f = new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/cube_desc/ssb.json"); CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class); @@ -73,11 +70,5 @@ public class FunctionRuleTest extends LocalFileMetadataTestCase { desc.getMeasures().add(measureDescDuplicated); desc.init(config); - ValidateContext vContext = new ValidateContext(); - rule.validate(desc, vContext); - - vContext.print(System.out); - assertTrue(vContext.getResults().length >= 1); - assertEquals("There is duplicated measure's name: " + measureDescDuplicated.getName(), vContext.getResults()[0].getMessage()); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java index 16086e3..4665465 100644 --- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java +++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java @@ -109,7 +109,7 @@ public class JoinedFlatTable { if (i > 0) { sql.append(","); } - sql.append(col.getTableAlias() + "." + col.getName() + "\n"); + sql.append(col.getExpressionInSourceDB() + "\n"); } appendJoinStatement(flatDesc, sql); appendWhereStatement(flatDesc, sql); @@ -149,7 +149,7 @@ public class JoinedFlatTable { if (i > 0) { sql.append(" AND "); } - sql.append(fk[i].getTableAlias() + "." + fk[i].getName() + " = " + pk[i].getTableAlias() + "." + pk[i].getName()); + sql.append(fk[i].getIdentity() + " = " + pk[i].getIdentity()); } sql.append("\n"); http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java index f17983f..b3ca14a 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java @@ -26,6 +26,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -40,6 +41,7 @@ import org.apache.kylin.metadata.cachesync.Broadcaster; import org.apache.kylin.metadata.cachesync.Broadcaster.Event; import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache; import org.apache.kylin.metadata.model.ColumnDesc; +import org.apache.kylin.metadata.model.ComputedColumnDesc; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.ExternalFilterDesc; import org.apache.kylin.metadata.model.TableDesc; @@ -114,6 +116,18 @@ public class MetadataManager { // name => External Filter Desc private CaseInsensitiveStringCache<ExternalFilterDesc> extFilterMap; + public static class CCInfo { + public ComputedColumnDesc computedColumnDesc; + public Set<DataModelDesc> dataModelDescs; + + public CCInfo(ComputedColumnDesc computedColumnDesc, Set<DataModelDesc> dataModelDescs) { + this.computedColumnDesc = computedColumnDesc; + this.dataModelDescs = dataModelDescs; + } + } + + private Map<String, CCInfo> ccInfoMap = Maps.newHashMap();// this is to check any two models won't conflict computed columns + private MetadataManager(KylinConfig config) throws IOException { init(config); } @@ -343,8 +357,9 @@ public class MetadataManager { @Override public void onProjectSchemaChange(Broadcaster broadcaster, String project) throws IOException { + ccInfoMap.clear(); for (String model : ProjectManager.getInstance(config).getProject(project).getModels()) { - reloadDataModelDesc(model); + reloadDataModelDescAt(DataModelDesc.concatResourcePath(model)); } } @@ -353,7 +368,7 @@ public class MetadataManager { if (event == Event.DROP) dataModelDescMap.removeLocal(cacheKey); else - reloadDataModelDesc(cacheKey); + reloadDataModelDescAt(DataModelDesc.concatResourcePath(cacheKey)); for (ProjectInstance prj : ProjectManager.getInstance(config).findProjectsByModel(cacheKey)) { broadcaster.notifyProjectSchemaUpdate(prj.getName()); @@ -568,7 +583,9 @@ public class MetadataManager { List<String> paths = store.collectResourceRecursively(ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT, MetadataConstants.FILE_SURFIX); for (String path : paths) { + try { + logger.info("Reloading data model at " + path); reloadDataModelDescAt(path); } catch (IllegalStateException e) { logger.error("Error to load DataModel at " + path, e); @@ -579,15 +596,11 @@ public class MetadataManager { logger.debug("Loaded " + dataModelDescMap.size() + " DataModel(s)"); } - public DataModelDesc reloadDataModelDesc(String name) { - return reloadDataModelDescAt(DataModelDesc.concatResourcePath(name)); - } - - private DataModelDesc reloadDataModelDescAt(String path) { + public DataModelDesc reloadDataModelDescAt(String path) { ResourceStore store = getStore(); try { DataModelDesc dataModelDesc = store.getResource(path, DataModelDesc.class, MODELDESC_SERIALIZER); - dataModelDesc.init(config, this.getAllTablesMap()); + dataModelDesc.init(config, this.getAllTablesMap(), this.ccInfoMap); dataModelDescMap.putLocal(dataModelDesc.getName(), dataModelDesc); return dataModelDesc; } catch (Exception e) { @@ -635,7 +648,7 @@ public class MetadataManager { } private DataModelDesc saveDataModelDesc(DataModelDesc dataModelDesc) throws IOException { - dataModelDesc.init(config, this.getAllTablesMap()); + dataModelDesc.init(config, this.getAllTablesMap(), this.ccInfoMap); String path = dataModelDesc.getResourcePath(); getStore().putResource(path, dataModelDesc, MODELDESC_SERIALIZER); http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java index e5b51e4..2cd4964 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java @@ -19,6 +19,7 @@ package org.apache.kylin.metadata.model; import java.io.Serializable; +import java.util.regex.Pattern; import org.apache.kylin.metadata.datatype.DataType; @@ -26,6 +27,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; /** * Column Metadata from Source. All name should be uppercase. @@ -37,10 +39,13 @@ public class ColumnDesc implements Serializable { @JsonProperty("id") private String id; + @JsonProperty("name") private String name; + @JsonProperty("datatype") private String datatype; + @JsonProperty("comment") @JsonInclude(JsonInclude.Include.NON_NULL) private String comment; @@ -62,10 +67,31 @@ public class ColumnDesc implements Serializable { private boolean isNullable = true; private TblColRef ref; + private String computedColumnExpr = null;//if null, it's not a computed column public ColumnDesc() { // default constructor for Jackson } + public ColumnDesc(ColumnDesc other) { + this.id = other.id; + this.name = other.name; + this.datatype = other.datatype; + this.dataGen = other.datatype; + this.comment = other.comment; + this.dataGen = other.dataGen; + this.index = other.index; + } + + public ColumnDesc(String id, String name, String datatype, String comment, String dataGen, String index, String computedColumnExpr) { + this.id = id; + this.name = name; + this.datatype = datatype; + this.comment = comment; + this.dataGen = dataGen; + this.index = index; + this.computedColumnExpr = computedColumnExpr; + } + /** Use TableRef.getColumn() instead */ @Deprecated public TblColRef getRef() { @@ -156,7 +182,7 @@ public class ColumnDesc implements Serializable { public void setNullable(boolean nullable) { this.isNullable = nullable; } - + public String getDataGen() { return dataGen; } @@ -165,6 +191,17 @@ public class ColumnDesc implements Serializable { return index; } + public String getComputedColumnExpr(String tableAlias, String tableIdentity) { + Preconditions.checkState(computedColumnExpr != null); + + //http://stackoverflow.com/questions/5054995/how-to-replace-case-insensitive-literal-substrings-in-java + return computedColumnExpr.replaceAll("(?i)" + Pattern.quote(tableIdentity), tableAlias); + } + + public boolean isComputedColumnn() { + return computedColumnExpr != null; + } + public void init(TableDesc table) { this.table = table; @@ -212,7 +249,7 @@ public class ColumnDesc implements Serializable { return false; } else if (!table.getIdentity().equals(other.table.getIdentity())) return false; - + return true; } http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java new file mode 100644 index 0000000..ac5830f --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +package org.apache.kylin.metadata.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; + +public class ComputedColumnDesc { + @JsonProperty + private String tableIdentity; + @JsonProperty + private String columnName; + @JsonProperty + private String expression; + @JsonProperty + private String datatype; + @JsonProperty + private String comment; + + public void init() { + Preconditions.checkNotNull(tableIdentity, "tableIdentity is null"); + Preconditions.checkNotNull(columnName, "columnName is null"); + Preconditions.checkNotNull(expression, "expression is null"); + Preconditions.checkNotNull(datatype, "datatype is null"); + + tableIdentity = tableIdentity.toUpperCase(); + columnName = columnName.toUpperCase(); + } + + public String getFullName() { + return tableIdentity + "." + columnName; + } + + public String getTableIdentity() { + return tableIdentity; + } + + public String getColumnName() { + return columnName; + } + + public String getExpression() { + return expression; + } + + public String getDatatype() { + return datatype; + } + + public String getComment() { + return comment; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + ComputedColumnDesc that = (ComputedColumnDesc) o; + + if (!tableIdentity.equals(that.tableIdentity)) + return false; + if (!columnName.equals(that.columnName)) + return false; + if (!expression.equals(that.expression)) + return false; + return datatype.equals(that.datatype); + } + + @Override + public int hashCode() { + int result = tableIdentity.hashCode(); + result = 31 * result + columnName.hashCode(); + result = 31 * result + expression.hashCode(); + result = 31 * result + datatype.hashCode(); + return result; + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java index 3669109..2174b7a 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java @@ -18,6 +18,8 @@ package org.apache.kylin.metadata.model; +import static org.apache.kylin.metadata.MetadataManager.CCInfo; + import java.io.Serializable; import java.util.ArrayDeque; import java.util.ArrayList; @@ -28,6 +30,9 @@ import java.util.Map; import java.util.Queue; import java.util.Set; +import javax.annotation.Nullable; + +import org.apache.commons.lang.mutable.MutableInt; import org.apache.commons.lang3.ArrayUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; @@ -42,6 +47,9 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.google.common.collect.FluentIterable; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -96,6 +104,10 @@ public class DataModelDesc extends RootPersistentEntity { @JsonProperty("capacity") private RealizationCapacity capacity = RealizationCapacity.MEDIUM; + @JsonProperty("computed_columns") + @JsonInclude(JsonInclude.Include.NON_DEFAULT) + private List<ComputedColumnDesc> computedColumnDescs = Lists.newArrayList(); + // computed attributes private TableRef rootFactTableRef; private Set<TableRef> factTableRefs = Sets.newLinkedHashSet(); @@ -291,7 +303,16 @@ public class DataModelDesc extends RootPersistentEntity { throw new IllegalArgumentException("Table not found by " + tableIdentity + " in model " + name); } - public void init(KylinConfig config, Map<String, TableDesc> tables) { + public void init(KylinConfig config, Map<String, TableDesc> originalTables, Map<String, CCInfo> ccInfoMap) { + //tweak the tables according to Computed Columns defined in model + Map<String, TableDesc> tables = Maps.newHashMap(); + for (Map.Entry<String, TableDesc> entry : originalTables.entrySet()) { + String s = entry.getKey(); + TableDesc tableDesc = entry.getValue(); + TableDesc extendedTableDesc = tableDesc.appendColumns(createComputedColumns(tableDesc)); + tables.put(s, extendedTableDesc); + } + this.config = config; initJoinTablesForUpgrade(); @@ -301,13 +322,32 @@ public class DataModelDesc extends RootPersistentEntity { initJoinsTree(); initDimensionsAndMetrics(); initPartitionDesc(); + initComputedColumns(ccInfoMap); boolean reinit = validate(); if (reinit) { // model slightly changed by validate() and must init() again - init(config, tables); + init(config, tables, ccInfoMap); } } + private ColumnDesc[] createComputedColumns(final TableDesc tableDesc) { + final MutableInt id = new MutableInt(tableDesc.getColumnCount()); + return FluentIterable.from(this.computedColumnDescs).filter(new Predicate<ComputedColumnDesc>() { + @Override + public boolean apply(@Nullable ComputedColumnDesc input) { + return tableDesc.getIdentity().equalsIgnoreCase(input.getTableIdentity()); + } + }).transform(new Function<ComputedColumnDesc, ColumnDesc>() { + @Nullable + @Override + public ColumnDesc apply(@Nullable ComputedColumnDesc input) { + id.increment(); + ColumnDesc columnDesc = new ColumnDesc(id.toString(), input.getColumnName(), input.getDatatype(), input.getComment(), null, null, input.getExpression()); + return columnDesc; + } + }).toArray(ColumnDesc.class); + } + private void initJoinTablesForUpgrade() { if (joinTables == null) { joinTables = new JoinTableDesc[0]; @@ -333,6 +373,7 @@ public class DataModelDesc extends RootPersistentEntity { TableDesc rootDesc = tables.get(rootFactTable); rootFactTableRef = new TableRef(this, rootDesc.getName(), rootDesc); + addAlias(rootFactTableRef); factTableRefs.add(rootFactTableRef); @@ -346,7 +387,9 @@ public class DataModelDesc extends RootPersistentEntity { String alias = join.getAlias(); if (alias == null) alias = tableDesc.getName(); + TableRef ref = new TableRef(this, alias, tableDesc); + join.setTableRef(ref); addAlias(ref); (join.getKind() == TableKind.LOOKUP ? lookupTableRefs : factTableRefs).add(ref); @@ -390,6 +433,30 @@ public class DataModelDesc extends RootPersistentEntity { this.partitionDesc.init(this); } + private void initComputedColumns(Map<String, CCInfo> ccInfoMap) { + Set<String> ccSet = Sets.newHashSet();//make sure cc name does not duplicate within this model + + for (ComputedColumnDesc computedColumnDesc : this.computedColumnDescs) { + computedColumnDesc.init(); + + if (ccSet.contains(computedColumnDesc.getFullName())) { + throw new IllegalArgumentException(String.format("More than one computed column named %s exist in model %s", computedColumnDesc.getFullName(), this.getName())); + } else { + ccSet.add(computedColumnDesc.getFullName()); + } + + CCInfo other = ccInfoMap.get(computedColumnDesc.getFullName()); + if (other == null || (other.dataModelDescs.size() == 1 && other.dataModelDescs.contains(this))) { + ccInfoMap.put(computedColumnDesc.getFullName(), new CCInfo(computedColumnDesc, Sets.<DataModelDesc> newHashSet(this))); + } else if (other.computedColumnDesc.equals(computedColumnDesc)) { + other.dataModelDescs.add(this); + } else { + throw new IllegalStateException(String.format("Computed column named %s is already defined in other models: %s. Please change another name, or try to keep consistent definition", // + computedColumnDesc.getFullName(), other.dataModelDescs)); + } + } + } + private void initJoinColumns() { for (JoinTableDesc joinTable : joinTables) { @@ -568,10 +635,8 @@ public class DataModelDesc extends RootPersistentEntity { } /** - * @param message - * error message - * @param silent - * if throw exception + * @param message error message + * @param silent if throw exception */ public void addError(String message, boolean silent) { if (!silent) { @@ -627,6 +692,10 @@ public class DataModelDesc extends RootPersistentEntity { return dimensions; } + public List<ComputedColumnDesc> getComputedColumnDescs() { + return computedColumnDescs; + } + public String[] getMetrics() { return metrics; } http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java index 8e2192f..7db730e 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java @@ -93,7 +93,7 @@ public class JoinsTree implements Serializable { boolean matches = false; if (chain.join == null) { - matches = anotherChain.join == null && chain.table.getTableDesc().equals(anotherChain.table.getTableDesc()); + matches = anotherChain.join == null && chain.table.getTableDesc().getIdentity().equals(anotherChain.table.getTableDesc().getIdentity()); } else { matches = chain.join.matches(anotherChain.join) && matchChain(chain.fkSide, anotherChain.fkSide, matchUp); } http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java index 38c2de5..3c00149 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java @@ -201,7 +201,7 @@ public class PartitionDesc implements Serializable { } private static void buildSingleColumnRangeCondAsTimeMillis(StringBuilder builder, TblColRef partitionColumn, long startInclusive, long endExclusive) { - String partitionColumnName = partitionColumn.getTableAlias() + "." + partitionColumn.getName(); + String partitionColumnName = partitionColumn.getIdentity(); if (startInclusive > 0) { builder.append(partitionColumnName + " >= " + startInclusive); builder.append(" AND "); @@ -210,7 +210,7 @@ public class PartitionDesc implements Serializable { } private static void buildSingleColumnRangeCondAsYmdInt(StringBuilder builder, TblColRef partitionColumn, long startInclusive, long endExclusive) { - String partitionColumnName = partitionColumn.getTableAlias() + "." + partitionColumn.getName(); + String partitionColumnName = partitionColumn.getIdentity(); if (startInclusive > 0) { builder.append(partitionColumnName + " >= " + DateFormat.formatToDateStr(startInclusive, DateFormat.COMPACT_DATE_PATTERN)); builder.append(" AND "); @@ -219,7 +219,7 @@ public class PartitionDesc implements Serializable { } private static void buildSingleColumnRangeCondition(StringBuilder builder, TblColRef partitionColumn, long startInclusive, long endExclusive, String partitionColumnDateFormat) { - String partitionColumnName = partitionColumn.getTableAlias() + "." + partitionColumn.getName(); + String partitionColumnName = partitionColumn.getIdentity(); if (startInclusive > 0) { builder.append(partitionColumnName + " >= '" + DateFormat.formatToDateStr(startInclusive, partitionColumnDateFormat) + "'"); builder.append(" AND "); @@ -228,8 +228,8 @@ public class PartitionDesc implements Serializable { } private static void buildMultipleColumnRangeCondition(StringBuilder builder, TblColRef partitionDateColumn, TblColRef partitionTimeColumn, long startInclusive, long endExclusive, String partitionColumnDateFormat, String partitionColumnTimeFormat) { - String partitionDateColumnName = partitionDateColumn.getTableAlias() + "." + partitionDateColumn.getName(); - String partitionTimeColumnName = partitionTimeColumn.getTableAlias() + "." + partitionTimeColumn.getName(); + String partitionDateColumnName = partitionDateColumn.getIdentity(); + String partitionTimeColumnName = partitionTimeColumn.getIdentity(); if (startInclusive > 0) { builder.append("("); builder.append("("); http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java index b4d49fb..b388f11 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java @@ -61,10 +61,34 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware { } public TableDesc(TableDesc other) { - this.name = other.getName(); - this.columns = other.getColumns(); + this.name = other.name; + this.sourceType = other.sourceType; this.database.setName(other.getDatabase()); - this.tableType = other.getTableType(); + this.tableType = other.tableType; + this.dataGen = other.dataGen; + this.columns = new ColumnDesc[other.columns.length]; + for (int i = 0; i < other.columns.length; i++) { + this.columns[i] = new ColumnDesc(other.columns[i]); + this.columns[i].init(this); + } + } + + public TableDesc appendColumns(ColumnDesc[] computedColumns) { + if (computedColumns == null || computedColumns.length == 0) { + return this; + } + + TableDesc ret = new TableDesc(this);//deep copy of the table desc + ColumnDesc[] origin = ret.columns; + ret.columns = new ColumnDesc[computedColumns.length + origin.length]; + for (int i = 0; i < origin.length; i++) { + ret.columns[i] = origin[i]; + } + for (int i = 0; i < computedColumns.length; i++) { + computedColumns[i].init(ret); + ret.columns[i + this.columns.length] = computedColumns[i]; + } + return ret; } public ColumnDesc findColumnByName(String name) { @@ -153,7 +177,12 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware { } public int getMaxColumnIndex() { + if (columns == null) { + return -1; + } + int max = -1; + for (ColumnDesc col : columns) { int idx = col.getZeroBasedIndex(); max = Math.max(max, idx); http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java index 6f15f3c..f4aec12 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java @@ -18,14 +18,14 @@ package org.apache.kylin.metadata.model; -import com.google.common.collect.Maps; - import java.io.Serializable; import java.util.Collection; import java.util.Collections; import java.util.Map; -public class TableRef implements Serializable{ +import com.google.common.collect.Maps; + +public class TableRef implements Serializable { final transient private DataModelDesc model; final private String alias; @@ -68,11 +68,11 @@ public class TableRef implements Serializable{ public TblColRef getColumn(String name) { return columns.get(name); } - + public Collection<TblColRef> getColumns() { return Collections.unmodifiableCollection(columns.values()); } - + // for test only @Deprecated public TblColRef makeFakeColumn(String name) { @@ -81,13 +81,13 @@ public class TableRef implements Serializable{ colDesc.setTable(table); return new TblColRef(this, colDesc); } - + // for test only @Deprecated public TblColRef makeFakeColumn(ColumnDesc colDesc) { return new TblColRef(this, colDesc); } - + @Override public boolean equals(Object o) { if (this == o) http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java index 4661bc9..da62a75 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java @@ -56,7 +56,7 @@ public class TblColRef implements Serializable { public static TblColRef newInnerColumn(String columnName, InnerDataTypeEnum dataType) { return newInnerColumn(columnName, dataType, null); } - + // used by projection rewrite, see OLAPProjectRel public static TblColRef newInnerColumn(String columnName, InnerDataTypeEnum dataType, String parserDescription) { ColumnDesc column = new ColumnDesc(); @@ -68,25 +68,25 @@ public class TblColRef implements Serializable { colRef.parserDescription = parserDescription; return colRef; } - + private static final DataModelDesc UNKNOWN_MODEL = new DataModelDesc(); static { UNKNOWN_MODEL.setName("UNKNOWN_MODEL"); } - + public static TableRef tableForUnknownModel(String tempTableAlias, TableDesc table) { return new TableRef(UNKNOWN_MODEL, tempTableAlias, table); } - + public static TblColRef columnForUnknownModel(TableRef table, ColumnDesc colDesc) { checkArgument(table.getModel() == UNKNOWN_MODEL); return new TblColRef(table, colDesc); } - + public static void fixUnknownModel(DataModelDesc model, String alias, TblColRef col) { checkArgument(col.table.getModel() == UNKNOWN_MODEL || col.table.getModel() == model); TableRef tableRef = model.findTable(alias); - checkArgument(tableRef.getTableDesc() == col.column.getTable()); + checkArgument(tableRef.getTableDesc().getIdentity().equals(col.column.getTable().getIdentity())); col.table = tableRef; col.identity = null; } @@ -101,7 +101,7 @@ public class TblColRef implements Serializable { desc.init(table); return new TblColRef(desc); } - + // ============================================================================ private TableRef table; @@ -112,13 +112,13 @@ public class TblColRef implements Serializable { TblColRef(ColumnDesc column) { this.column = column; } - + TblColRef(TableRef table, ColumnDesc column) { - checkArgument(table.getTableDesc() == column.getTable()); + checkArgument(table.getTableDesc().getIdentity().equals(column.getTable().getIdentity())); this.table = table; this.column = column; } - + public ColumnDesc getColumnDesc() { return column; } @@ -130,15 +130,23 @@ public class TblColRef implements Serializable { public TableRef getTableRef() { return table; } - + public boolean isQualified() { return table != null; } - + public String getTableAlias() { return table != null ? table.getAlias() : "UNKNOWN_ALIAS"; } - + + public String getExpressionInSourceDB() { + if (!column.isComputedColumnn()) { + return getIdentity(); + } else { + return column.getComputedColumnExpr(getTableAlias(), table.getTableIdentity()); + } + } + public String getTable() { if (column.getTable() == null) { return null; @@ -207,7 +215,7 @@ public class TblColRef implements Serializable { public String toString() { if (isInnerColumn() && parserDescription != null) return parserDescription; - + String alias = table == null ? "UNKNOWN_MODEL" : table.getAlias(); String tableName = column.getTable() == null ? "NULL" : column.getTable().getName(); String tableIdentity = column.getTable() == null ? "NULL" : column.getTable().getIdentity(); http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java index 63188a1..deac12d 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java @@ -242,10 +242,15 @@ class ProjectL2Cache { logger.error("Realization '" + realization.getCanonicalName() + "' reports column '" + col.getCanonicalName() + "', but its table is not found by MetadataManager"); return false; } - ColumnDesc foundCol = table.findColumnByName(col.getName()); - if (col.getColumnDesc().equals(foundCol) == false) { - logger.error("Realization '" + realization.getCanonicalName() + "' reports column '" + col.getCanonicalName() + "', but it is not equal to '" + foundCol + "' according to MetadataManager"); - return false; + + if (!col.getColumnDesc().isComputedColumnn()) { + ColumnDesc foundCol = table.findColumnByName(col.getName()); + if (col.getColumnDesc().equals(foundCol) == false) { + logger.error("Realization '" + realization.getCanonicalName() + "' reports column '" + col.getCanonicalName() + "', but it is not equal to '" + foundCol + "' according to MetadataManager"); + return false; + } + } else { + //computed column may not exit here } // auto-define table required by realization for some legacy test case http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java index b319423..f6d6fc3 100644 --- a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java @@ -19,6 +19,7 @@ package org.apache.kylin.metadata.model; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import org.apache.commons.lang.ArrayUtils; @@ -41,7 +42,19 @@ public class DataModelDescTest extends LocalFileMetadataTestCase { public void after() throws Exception { this.cleanupTestMetadata(); } - + + @Test + public void loadInnerModel() { + DataModelDesc model = MetadataManager.getInstance(getTestConfig()).getDataModelDesc("ci_inner_join_model"); + assertNotNull(model); + } + + @Test + public void loadLeftModel() { + DataModelDesc model = MetadataManager.getInstance(getTestConfig()).getDataModelDesc("ci_left_join_model"); + assertNotNull(model); + } + @Test public void testNoDupColInDimAndMeasure() { DataModelDesc model = MetadataManager.getInstance(getTestConfig()).getDataModelDesc("test_kylin_inner_join_model_desc"); http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json index cfaadac..3747af8 100644 --- a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json +++ b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json @@ -1,445 +1,615 @@ { - "uuid" : "3819ad72-3929-4dff-b59d-cd89a01238af", - "name" : "ci_inner_join_cube", - "model_name" : "ci_inner_join_model", - "description" : null, - "dimensions" : [ { - "name" : "CAL_DT", - "table" : "TEST_CAL_DT", - "column" : "{FK}", - "derived" : [ "WEEK_BEG_DT" ] - }, { - "name" : "ORDER_ID", - "table" : "TEST_KYLIN_FACT", - "column" : "ORDER_ID" - }, { - "name" : "TEST_DATE_ENC", - "table" : "TEST_ORDER", - "column" : "TEST_DATE_ENC" - }, { - "name" : "TEST_TIME_ENC", - "table" : "TEST_ORDER", - "column" : "TEST_TIME_ENC" - }, { - "name" : "CATEGORY", - "table" : "TEST_CATEGORY_GROUPINGS", - "column" : "{FK}", - "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ] - }, { - "name" : "CATEGORY_HIERARCHY", - "table" : "TEST_CATEGORY_GROUPINGS", - "column" : "META_CATEG_NAME", - "derived" : null - }, { - "name" : "CATEGORY_HIERARCHY", - "table" : "TEST_CATEGORY_GROUPINGS", - "column" : "CATEG_LVL2_NAME", - "derived" : null - }, { - "name" : "CATEGORY_HIERARCHY", - "table" : "TEST_CATEGORY_GROUPINGS", - "column" : "CATEG_LVL3_NAME", - "derived" : null - }, { - "name" : "LSTG_FORMAT_NAME", - "table" : "TEST_KYLIN_FACT", - "column" : "LSTG_FORMAT_NAME", - "derived" : null - }, { - "name" : "SITE_ID", - "table" : "TEST_SITES", - "column" : "{FK}", - "derived" : [ "SITE_NAME", "CRE_USER" ] - }, { - "name" : "SELLER_TYPE_CD", - "table" : "TEST_SELLER_TYPE_DIM", - "column" : "{FK}", - "derived" : [ "SELLER_TYPE_DESC" ] - }, { - "name" : "SELLER_ID", - "table" : "TEST_KYLIN_FACT", - "column" : "SELLER_ID" - }, { - "name" : "SELLER_BUYER_LEVEL", - "table" : "SELLER_ACCOUNT", - "column" : "ACCOUNT_BUYER_LEVEL" - }, { - "name" : "SELLER_SELLER_LEVEL", - "table" : "SELLER_ACCOUNT", - "column" : "ACCOUNT_SELLER_LEVEL" - }, { - "name" : "SELLER_COUNTRY", - "table" : "SELLER_ACCOUNT", - "column" : "ACCOUNT_COUNTRY" - }, { - "name" : "SELLER_COUNTRY_NAME", - "table" : "SELLER_COUNTRY", - "column" : "NAME" - }, { - "name" : "BUYER_ID", - "table" : "TEST_ORDER", - "column" : "BUYER_ID" - }, { - "name" : "BUYER_BUYER_LEVEL", - "table" : "BUYER_ACCOUNT", - "column" : "ACCOUNT_BUYER_LEVEL" - }, { - "name" : "BUYER_SELLER_LEVEL", - "table" : "BUYER_ACCOUNT", - "column" : "ACCOUNT_SELLER_LEVEL" - }, { - "name" : "BUYER_COUNTRY", - "table" : "BUYER_ACCOUNT", - "column" : "ACCOUNT_COUNTRY" - }, { - "name" : "BUYER_COUNTRY_NAME", - "table" : "BUYER_COUNTRY", - "column" : "NAME" - } ], - "measures" : [ { - "name" : "TRANS_CNT", - "function" : { - "expression" : "COUNT", - "parameter" : { - "type" : "constant", - "value" : "1" - }, - "returntype" : "bigint" + "uuid": "3819ad72-3929-4dff-b59d-cd89a01238af", + "name": "ci_inner_join_cube", + "model_name": "ci_inner_join_model", + "description": null, + "dimensions": [ + { + "name": "CAL_DT", + "table": "TEST_CAL_DT", + "column": "{FK}", + "derived": [ + "WEEK_BEG_DT" + ] + }, + { + "name": "ORDER_ID", + "table": "TEST_KYLIN_FACT", + "column": "ORDER_ID" + }, + { + "name": "TEST_DATE_ENC", + "table": "TEST_ORDER", + "column": "TEST_DATE_ENC" + }, + { + "name": "TEST_TIME_ENC", + "table": "TEST_ORDER", + "column": "TEST_TIME_ENC" + }, + { + "name": "CATEGORY", + "table": "TEST_CATEGORY_GROUPINGS", + "column": "{FK}", + "derived": [ + "USER_DEFINED_FIELD1", + "USER_DEFINED_FIELD3", + "UPD_DATE", + "UPD_USER" + ] + }, + { + "name": "CATEGORY_HIERARCHY", + "table": "TEST_CATEGORY_GROUPINGS", + "column": "META_CATEG_NAME", + "derived": null + }, + { + "name": "CATEGORY_HIERARCHY", + "table": "TEST_CATEGORY_GROUPINGS", + "column": "CATEG_LVL2_NAME", + "derived": null + }, + { + "name": "CATEGORY_HIERARCHY", + "table": "TEST_CATEGORY_GROUPINGS", + "column": "CATEG_LVL3_NAME", + "derived": null + }, + { + "name": "LSTG_FORMAT_NAME", + "table": "TEST_KYLIN_FACT", + "column": "LSTG_FORMAT_NAME", + "derived": null + }, + { + "name": "SITE_ID", + "table": "TEST_SITES", + "column": "{FK}", + "derived": [ + "SITE_NAME", + "CRE_USER" + ] + }, + { + "name": "SELLER_TYPE_CD", + "table": "TEST_SELLER_TYPE_DIM", + "column": "{FK}", + "derived": [ + "SELLER_TYPE_DESC" + ] + }, + { + "name": "SELLER_ID", + "table": "TEST_KYLIN_FACT", + "column": "SELLER_ID" + }, + { + "name": "SELLER_BUYER_LEVEL", + "table": "SELLER_ACCOUNT", + "column": "ACCOUNT_BUYER_LEVEL" + }, + { + "name": "SELLER_SELLER_LEVEL", + "table": "SELLER_ACCOUNT", + "column": "ACCOUNT_SELLER_LEVEL" + }, + { + "name": "SELLER_COUNTRY", + "table": "SELLER_ACCOUNT", + "column": "ACCOUNT_COUNTRY" + }, + { + "name": "SELLER_COUNTRY_NAME", + "table": "SELLER_COUNTRY", + "column": "NAME" + }, + { + "name": "BUYER_ID", + "table": "TEST_ORDER", + "column": "BUYER_ID" + }, + { + "name": "BUYER_BUYER_LEVEL", + "table": "BUYER_ACCOUNT", + "column": "ACCOUNT_BUYER_LEVEL" + }, + { + "name": "BUYER_SELLER_LEVEL", + "table": "BUYER_ACCOUNT", + "column": "ACCOUNT_SELLER_LEVEL" + }, + { + "name": "BUYER_COUNTRY", + "table": "BUYER_ACCOUNT", + "column": "ACCOUNT_COUNTRY" + }, + { + "name": "DEAL_YEAR", + "table": "TEST_KYLIN_FACT", + "column": "DEAL_YEAR" + }, + { + "name": "SELLER_COUNTRY_ABBR", + "table": "SELLER_ACCOUNT", + "column": "COUNTRY_ABBR" + }, + { + "name": "BUYER_COUNTRY_ABBR", + "table": "BUYER_ACCOUNT", + "column": "COUNTRY_ABBR" + }, + { + "name": "BUYER_COUNTRY_NAME", + "table": "BUYER_COUNTRY", + "column": "NAME" } - }, { - "name" : "ITEM_COUNT_SUM", - "function" : { - "expression" : "SUM", - "parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.ITEM_COUNT" - }, - "returntype" : "bigint" + ], + "measures": [ + { + "name": "TRANS_CNT", + "function": { + "expression": "COUNT", + "parameter": { + "type": "constant", + "value": "1" + }, + "returntype": "bigint" + } + }, + { + "name": "ITEM_COUNT_SUM", + "function": { + "expression": "SUM", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.ITEM_COUNT" + }, + "returntype": "bigint" + } + }, + { + "name": "GMV_SUM", + "function": { + "expression": "SUM", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.PRICE" + }, + "returntype": "decimal(19,4)" + } + }, + { + "name": "GMV_MIN", + "function": { + "expression": "MIN", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.PRICE" + }, + "returntype": "decimal(19,4)" + } + }, + { + "name": "GMV_MAX", + "function": { + "expression": "MAX", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.PRICE" + }, + "returntype": "decimal(19,4)" + } + }, + { + "name": "SELLER_HLL", + "function": { + "expression": "COUNT_DISTINCT", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.SELLER_ID" + }, + "returntype": "hllc(10)" + } + }, + { + "name": "SELLER_FORMAT_HLL", + "function": { + "expression": "COUNT_DISTINCT", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "next_parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.SELLER_ID" + } + }, + "returntype": "hllc(10)" + } + }, + { + "name": "TOP_SELLER", + "function": { + "expression": "TOP_N", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.PRICE", + "next_parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.SELLER_ID" + } + }, + "returntype": "topn(100, 4)", + "configuration": { + "topn.encoding.TEST_KYLIN_FACT.SELLER_ID": "int:4" + } + } + }, + { + "name": "TEST_COUNT_DISTINCT_BITMAP", + "function": { + "expression": "COUNT_DISTINCT", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP" + }, + "returntype": "bitmap" + } + }, + { + "name": "TEST_EXTENDED_COLUMN", + "function": { + "expression": "EXTENDED_COLUMN", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.ORDER_ID", + "next_parameter": { + "type": "column", + "value": "TEST_ORDER.TEST_EXTENDED_COLUMN" + } + }, + "returntype": "extendedcolumn(100)" + } + }, + { + "name": "BUYER_CONTACT", + "function": { + "expression": "EXTENDED_COLUMN", + "parameter": { + "type": "column", + "value": "TEST_ORDER.BUYER_ID", + "next_parameter": { + "type": "column", + "value": "BUYER_ACCOUNT.ACCOUNT_CONTACT" + } + }, + "returntype": "extendedcolumn(100)" + } + }, + { + "name": "SELLER_CONTACT", + "function": { + "expression": "EXTENDED_COLUMN", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.SELLER_ID", + "next_parameter": { + "type": "column", + "value": "SELLER_ACCOUNT.ACCOUNT_CONTACT" + } + }, + "returntype": "extendedcolumn(100)" + } + }, + { + "name": "TRANS_ID_RAW", + "function": { + "expression": "RAW", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.TRANS_ID" + }, + "returntype": "raw" + } + }, + { + "name": "PRICE_RAW", + "function": { + "expression": "RAW", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.PRICE" + }, + "returntype": "raw" + } + }, + { + "name": "CAL_DT_RAW", + "function": { + "expression": "RAW", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.CAL_DT" + }, + "returntype": "raw" + } + }, + { + "name": "COMPUTED_COLUMN_MEASURE", + "function": { + "expression": "SUM", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.DEAL_AMOUNT" + }, + "returntype": "decimal" + } } - }, { - "name" : "GMV_SUM", - "function" : { - "expression" : "SUM", - "parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.PRICE" - }, - "returntype" : "decimal(19,4)" + ], + "dictionaries": [ + { + "column": "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP", + "builder": "org.apache.kylin.dict.GlobalDictionaryBuilder" } - }, { - "name" : "GMV_MIN", - "function" : { - "expression" : "MIN", - "parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.PRICE" + ], + "rowkey": { + "rowkey_columns": [ + { + "column": "TEST_KYLIN_FACT.SELLER_ID", + "encoding": "int:4" }, - "returntype" : "decimal(19,4)" - } - }, { - "name" : "GMV_MAX", - "function" : { - "expression" : "MAX", - "parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.PRICE" + { + "column": "TEST_KYLIN_FACT.ORDER_ID", + "encoding": "int:4" }, - "returntype" : "decimal(19,4)" - } - }, { - "name" : "SELLER_HLL", - "function" : { - "expression" : "COUNT_DISTINCT", - "parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.SELLER_ID" + { + "column": "TEST_KYLIN_FACT.CAL_DT", + "encoding": "dict" }, - "returntype" : "hllc(10)" - } - }, { - "name" : "SELLER_FORMAT_HLL", - "function" : { - "expression" : "COUNT_DISTINCT", - "parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", - "next_parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.SELLER_ID" - } + { + "column": "TEST_KYLIN_FACT.LEAF_CATEG_ID", + "encoding": "dict" }, - "returntype" : "hllc(10)" - } - }, { - "name" : "TOP_SELLER", - "function" : { - "expression" : "TOP_N", - "parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.PRICE", - "next_parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.SELLER_ID" - } + { + "column": "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", + "encoding": "dict" }, - "returntype" : "topn(100, 4)", - "configuration": {"topn.encoding.TEST_KYLIN_FACT.SELLER_ID" : "int:4"} - } - }, { - "name" : "TEST_COUNT_DISTINCT_BITMAP", - "function" : { - "expression" : "COUNT_DISTINCT", - "parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP" + { + "column": "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "encoding": "dict" }, - "returntype" : "bitmap" - } - }, { - "name" : "TEST_EXTENDED_COLUMN", - "function" : { - "expression" : "EXTENDED_COLUMN", - "parameter": { - "type": "column", - "value": "TEST_KYLIN_FACT.ORDER_ID", - "next_parameter": { - "type": "column", - "value": "TEST_ORDER.TEST_EXTENDED_COLUMN" - } + { + "column": "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + "encoding": "dict" }, - "returntype": "extendedcolumn(100)" - } - }, { - "name" : "BUYER_CONTACT", - "function" : { - "expression" : "EXTENDED_COLUMN", - "parameter": { - "type": "column", - "value": "TEST_ORDER.BUYER_ID", - "next_parameter": { - "type": "column", - "value": "BUYER_ACCOUNT.ACCOUNT_CONTACT" - } + { + "column": "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "encoding": "fixed_length:12" }, - "returntype": "extendedcolumn(100)" - } - }, { - "name" : "SELLER_CONTACT", - "function" : { - "expression" : "EXTENDED_COLUMN", - "parameter": { - "type": "column", - "value": "TEST_KYLIN_FACT.SELLER_ID", - "next_parameter": { - "type": "column", - "value": "SELLER_ACCOUNT.ACCOUNT_CONTACT" - } + { + "column": "TEST_KYLIN_FACT.LSTG_SITE_ID", + "encoding": "dict" }, - "returntype": "extendedcolumn(100)" - } - }, { - "name" : "TRANS_ID_RAW", - "function" : { - "expression" : "RAW", - "parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.TRANS_ID" + { + "column": "TEST_KYLIN_FACT.SLR_SEGMENT_CD", + "encoding": "dict" }, - "returntype" : "raw" - } - }, { - "name" : "PRICE_RAW", - "function" : { - "expression" : "RAW", - "parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.PRICE" + { + "column": "TEST_ORDER.TEST_TIME_ENC", + "encoding": "time" }, - "returntype" : "raw" - } - }, { - "name" : "CAL_DT_RAW", - "function" : { - "expression" : "RAW", - "parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.CAL_DT" + { + "column": "TEST_ORDER.TEST_DATE_ENC", + "encoding": "date" }, - "returntype" : "raw" - } - } ], - "dictionaries": [ { - "column": "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP", - "builder": "org.apache.kylin.dict.GlobalDictionaryBuilder" - } ], - "rowkey" : { - "rowkey_columns" : [ { - "column" : "TEST_KYLIN_FACT.SELLER_ID", - "encoding" : "int:4" - }, { - "column" : "TEST_KYLIN_FACT.ORDER_ID", - "encoding" : "int:4" - }, { - "column" : "TEST_KYLIN_FACT.CAL_DT", - "encoding" : "dict" - }, { - "column" : "TEST_KYLIN_FACT.LEAF_CATEG_ID", - "encoding" : "dict" - }, { - "column" : "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", - "encoding" : "dict" - }, { - "column" : "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", - "encoding" : "dict" - }, { - "column" : "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", - "encoding" : "dict" - }, { - "column" : "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", - "encoding" : "fixed_length:12" - }, { - "column" : "TEST_KYLIN_FACT.LSTG_SITE_ID", - "encoding" : "dict" - }, { - "column" : "TEST_KYLIN_FACT.SLR_SEGMENT_CD", - "encoding" : "dict" - }, { - "column" : "TEST_ORDER.TEST_TIME_ENC", - "encoding" : "time" - }, { - "column" : "TEST_ORDER.TEST_DATE_ENC", - "encoding" : "date" - }, { - "column" : "TEST_ORDER.BUYER_ID", - "encoding" : "int:4" - }, { - "column" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", - "encoding" : "dict" - }, { - "column" : "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL", - "encoding" : "dict" - }, { - "column" : "BUYER_ACCOUNT.ACCOUNT_COUNTRY", - "encoding" : "dict" - }, { - "column" : "BUYER_COUNTRY.NAME", - "encoding" : "dict" - }, { - "column" : "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL", - "encoding" : "dict" - }, { - "column" : "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", - "encoding" : "dict" - }, { - "column" : "SELLER_ACCOUNT.ACCOUNT_COUNTRY", - "encoding" : "dict" - }, { - "column" : "SELLER_COUNTRY.NAME", - "encoding" : "dict" - } ] + { + "column": "TEST_ORDER.BUYER_ID", + "encoding": "int:4" + }, + { + "column": "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "encoding": "dict" + }, + { + "column": "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "encoding": "dict" + }, + { + "column": "BUYER_ACCOUNT.ACCOUNT_COUNTRY", + "encoding": "dict" + }, + { + "column": "BUYER_COUNTRY.NAME", + "encoding": "dict" + }, + { + "column": "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "encoding": "dict" + }, + { + "column": "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "encoding": "dict" + }, + { + "column": "SELLER_ACCOUNT.ACCOUNT_COUNTRY", + "encoding": "dict" + }, + { + "column": "TEST_KYLIN_FACT.DEAL_YEAR", + "encoding": "int:2" + }, + { + "column": "SELLER_COUNTRY.NAME", + "encoding": "dict" + }, + { + "column": "SELLER_ACCOUNT.COUNTRY_ABBR", + "encoding": "dict" + }, + { + "column": "BUYER_ACCOUNT.COUNTRY_ABBR", + "encoding": "dict" + } + ] }, - "signature" : null, - "last_modified" : 1448959801311, - "null_string" : null, - "hbase_mapping" : { - "column_family" : [ { - "name" : "f1", - "columns" : [ { - "qualifier" : "m", - "measure_refs" : [ "TRANS_CNT", "ITEM_COUNT_SUM", "GMV_SUM", "GMV_MIN", "GMV_MAX" ] - } ] - }, { - "name" : "f2", - "columns" : [ { - "qualifier" : "m", - "measure_refs" : [ "SELLER_HLL", "SELLER_FORMAT_HLL", "TOP_SELLER", "TEST_COUNT_DISTINCT_BITMAP" ] - } ] - }, { - "name" : "f3", - "columns" : [ { - "qualifier" : "m", - "measure_refs" : [ "TEST_EXTENDED_COLUMN", "TRANS_ID_RAW", "PRICE_RAW", "CAL_DT_RAW", "BUYER_CONTACT", "SELLER_CONTACT" ] - } ] - } ] + "signature": null, + "last_modified": 1448959801311, + "null_string": null, + "hbase_mapping": { + "column_family": [ + { + "name": "f1", + "columns": [ + { + "qualifier": "m", + "measure_refs": [ + "TRANS_CNT", + "ITEM_COUNT_SUM", + "GMV_SUM", + "GMV_MIN", + "GMV_MAX", + "COMPUTED_COLUMN_MEASURE" + ] + } + ] + }, + { + "name": "f2", + "columns": [ + { + "qualifier": "m", + "measure_refs": [ + "SELLER_HLL", + "SELLER_FORMAT_HLL", + "TOP_SELLER", + "TEST_COUNT_DISTINCT_BITMAP" + ] + } + ] + }, + { + "name": "f3", + "columns": [ + { + "qualifier": "m", + "measure_refs": [ + "TEST_EXTENDED_COLUMN", + "TRANS_ID_RAW", + "PRICE_RAW", + "CAL_DT_RAW", + "BUYER_CONTACT", + "SELLER_CONTACT" + ] + } + ] + } + ] }, - "aggregation_groups" : [ { - "includes" : [ "TEST_KYLIN_FACT.CAL_DT", - "TEST_KYLIN_FACT.LEAF_CATEG_ID", - "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", - "TEST_KYLIN_FACT.LSTG_SITE_ID", - "TEST_KYLIN_FACT.SLR_SEGMENT_CD", - "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", - "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", - "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME" ], - "select_rule" : { - "hierarchy_dims" : [ [ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", - "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", - "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", - "TEST_KYLIN_FACT.LEAF_CATEG_ID" ] ], - "mandatory_dims" : [ ], - "joint_dims" : [ [ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", - "TEST_KYLIN_FACT.LSTG_SITE_ID", - "TEST_KYLIN_FACT.SLR_SEGMENT_CD" ] ] - } - }, { - "includes" : [ "TEST_KYLIN_FACT.CAL_DT", - "TEST_KYLIN_FACT.LEAF_CATEG_ID", - "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", - "TEST_KYLIN_FACT.LSTG_SITE_ID", - "TEST_KYLIN_FACT.SLR_SEGMENT_CD", - "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", - "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", - "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", - - "TEST_KYLIN_FACT.SELLER_ID", - "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL", - "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", - "SELLER_ACCOUNT.ACCOUNT_COUNTRY", - "SELLER_COUNTRY.NAME", - - "TEST_KYLIN_FACT.ORDER_ID", - "TEST_ORDER.TEST_DATE_ENC", - "TEST_ORDER.TEST_TIME_ENC", - "TEST_ORDER.BUYER_ID", - "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", - "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL", - "BUYER_ACCOUNT.ACCOUNT_COUNTRY", - "BUYER_COUNTRY.NAME" ], - "select_rule" : { - "hierarchy_dims" : [ ], - "mandatory_dims" : [ "TEST_KYLIN_FACT.CAL_DT" ], - - "joint_dims" : [ [ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", - "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", - "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", - "TEST_KYLIN_FACT.LEAF_CATEG_ID" ], - - [ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", - "TEST_KYLIN_FACT.LSTG_SITE_ID", - "TEST_KYLIN_FACT.SLR_SEGMENT_CD" ], - - [ "TEST_KYLIN_FACT.SELLER_ID", - "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL", - "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", - "SELLER_ACCOUNT.ACCOUNT_COUNTRY", - "SELLER_COUNTRY.NAME" ], - - [ "TEST_KYLIN_FACT.ORDER_ID", - "TEST_ORDER.TEST_DATE_ENC", - "TEST_ORDER.TEST_TIME_ENC", - "TEST_ORDER.BUYER_ID", - "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", - "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL", - "BUYER_ACCOUNT.ACCOUNT_COUNTRY", - "BUYER_COUNTRY.NAME" ] ] + "aggregation_groups": [ + { + "includes": [ + "TEST_KYLIN_FACT.CAL_DT", + "TEST_KYLIN_FACT.LEAF_CATEG_ID", + "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "TEST_KYLIN_FACT.LSTG_SITE_ID", + "TEST_KYLIN_FACT.SLR_SEGMENT_CD", + "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + "TEST_KYLIN_FACT.DEAL_YEAR" + ], + "select_rule": { + "hierarchy_dims": [ + [ + "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + "TEST_KYLIN_FACT.LEAF_CATEG_ID" + ] + ], + "mandatory_dims": [], + "joint_dims": [ + [ + "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "TEST_KYLIN_FACT.LSTG_SITE_ID", + "TEST_KYLIN_FACT.SLR_SEGMENT_CD", + "TEST_KYLIN_FACT.DEAL_YEAR" + ] + ] + } + }, + { + "includes": [ + "TEST_KYLIN_FACT.CAL_DT", + "TEST_KYLIN_FACT.LEAF_CATEG_ID", + "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "TEST_KYLIN_FACT.LSTG_SITE_ID", + "TEST_KYLIN_FACT.SLR_SEGMENT_CD", + "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + "TEST_KYLIN_FACT.SELLER_ID", + "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "SELLER_ACCOUNT.ACCOUNT_COUNTRY", + "SELLER_COUNTRY.NAME", + "TEST_KYLIN_FACT.ORDER_ID", + "TEST_ORDER.TEST_DATE_ENC", + "TEST_ORDER.TEST_TIME_ENC", + "TEST_ORDER.BUYER_ID", + "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "BUYER_ACCOUNT.ACCOUNT_COUNTRY", + "BUYER_COUNTRY.NAME", + "SELLER_ACCOUNT.COUNTRY_ABBR", + "BUYER_ACCOUNT.COUNTRY_ABBR" + ], + "select_rule": { + "hierarchy_dims": [], + "mandatory_dims": [ + "TEST_KYLIN_FACT.CAL_DT" + ], + "joint_dims": [ + [ + "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + "TEST_KYLIN_FACT.LEAF_CATEG_ID" + ], + [ + "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "TEST_KYLIN_FACT.LSTG_SITE_ID", + "TEST_KYLIN_FACT.SLR_SEGMENT_CD", + "SELLER_ACCOUNT.COUNTRY_ABBR", + "BUYER_ACCOUNT.COUNTRY_ABBR" + ], + [ + "TEST_KYLIN_FACT.SELLER_ID", + "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "SELLER_ACCOUNT.ACCOUNT_COUNTRY", + "SELLER_COUNTRY.NAME" + ], + [ + "TEST_KYLIN_FACT.ORDER_ID", + "TEST_ORDER.TEST_DATE_ENC", + "TEST_ORDER.TEST_TIME_ENC", + "TEST_ORDER.BUYER_ID", + "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "BUYER_ACCOUNT.ACCOUNT_COUNTRY", + "BUYER_COUNTRY.NAME" + ] + ] + } } - } ], - "notify_list" : null, - "status_need_notify" : [ ], - "auto_merge_time_ranges" : null, - "retention_range" : 0, - "engine_type" : 4, - "storage_type" : 2, + ], + "notify_list": null, + "status_need_notify": [], + "auto_merge_time_ranges": null, + "retention_range": 0, + "engine_type": 2, + "storage_type": 2, "override_kylin_properties": { "kylin.cube.algorithm": "LAYER" },
