KYLIN-867 refactor Hybrid
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/4f742cda Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/4f742cda Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/4f742cda Branch: refs/heads/0.7-staging Commit: 4f742cda2f996b7b744f98e0f8515ba74f036299 Parents: 9491e92 Author: shaofengshi <[email protected]> Authored: Fri Jul 3 16:43:42 2015 +0800 Committer: shaofengshi <[email protected]> Committed: Mon Jul 6 16:19:17 2015 +0800 ---------------------------------------------------------------------- .../common/util/AbstractKylinTestCase.java | 2 +- .../org/apache/kylin/cube/CubeInstance.java | 5 - .../hybrid/test_kylin_hybrid_inner_join.json | 15 +- .../hybrid/test_kylin_hybrid_left_join.json | 15 +- .../localmeta/project/default.json | 10 ++ .../apache/kylin/invertedindex/IIInstance.java | 5 - .../metadata/project/RealizationEntry.java | 22 ++- .../metadata/realization/IRealization.java | 2 - .../AdjustForWeeklyMatchedRealization.java | 4 +- .../RoutingRules/RealizationPriorityRule.java | 4 +- .../apache/kylin/query/test/IIQueryTest.java | 12 +- .../apache/kylin/query/test/KylinQueryTest.java | 6 + .../apache/kylin/rest/service/BasicService.java | 5 + .../apache/kylin/rest/service/CacheService.java | 4 + .../apache/kylin/rest/service/CubeService.java | 6 +- .../kylin/storage/hybrid/HybridInstance.java | 161 +++++++++++-------- .../kylin/storage/hybrid/HybridManager.java | 25 ++- .../storage/hybrid/HybridStorageEngine.java | 62 +++---- .../kylin/storage/hybrid/HybridManagerTest.java | 9 +- 19 files changed, 207 insertions(+), 167 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java b/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java index a4a6dc4..d337501 100644 --- a/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java +++ b/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java @@ -36,7 +36,7 @@ public abstract class AbstractKylinTestCase { public static final String SANDBOX_TEST_DATA = "../examples/test_case_data/sandbox"; - public static final String[] SERVICES_WITH_CACHE = { "org.apache.kylin.metadata.MetadataManager", "org.apache.kylin.cube.CubeManager", "org.apache.kylin.cube.CubeDescManager", "org.apache.kylin.invertedindex.IIDescManager", "org.apache.kylin.invertedindex.IIManager", "org.apache.kylin.metadata.realization.RealizationRegistry", "org.apache.kylin.metadata.project.ProjectManager" }; + public static final String[] SERVICES_WITH_CACHE = { "org.apache.kylin.metadata.MetadataManager", "org.apache.kylin.cube.CubeManager", "org.apache.kylin.cube.CubeDescManager", "org.apache.kylin.invertedindex.IIDescManager", "org.apache.kylin.invertedindex.IIManager", "org.apache.kylin.metadata.realization.RealizationRegistry", "org.apache.kylin.storage.hybrid.HybridManager", "org.apache.kylin.metadata.project.ProjectManager" }; public abstract void createTestMetadata() throws Exception; http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java ---------------------------------------------------------------------- diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java index 898925a..88f110f 100644 --- a/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java +++ b/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java @@ -377,11 +377,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization { } @Override - public String getModelName() { - return this.getDescriptor().getModelName(); - } - - @Override public List<TblColRef> getAllDimensions() { return Lists.newArrayList(getDescriptor().listDimensionColumnsIncludingDerived()); } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json index 5582841..e38e55b 100644 --- a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json +++ b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json @@ -1,14 +1,13 @@ { "uuid": "9iiu8590-64b6-4367-8fb5-7500eb95fd9c", "name": "test_kylin_hybrid_inner_join", - "historyRealization": { - "type": "CUBE", - "realization": "test_kylin_cube_without_slr_empty" - }, - "realTimeRealization": { - "type": "INVERTED_INDEX", - "realization": "test_kylin_ii" - }, + "realizations": [ + { + "type": "CUBE", + "realization": "test_kylin_cube_without_slr_empty" + } + ], + "cost": 40, "last_modified": 1420016227424, "create_time": null } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json index 37f59d6..48cebe6 100644 --- a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json +++ b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json @@ -1,14 +1,13 @@ { "uuid": "5ca78590-64b6-4367-8fb5-7500eb95fd9c", "name": "test_kylin_hybrid_left_join", - "historyRealization": { - "type": "CUBE", - "realization": "test_kylin_cube_with_slr_left_join_empty" - }, - "realTimeRealization": { - "type": "INVERTED_INDEX", - "realization": "test_kylin_ii" - }, + "realizations": [ + { + "type": "CUBE", + "realization": "test_kylin_cube_without_slr_left_join_empty" + } + ], + "cost": 40, "last_modified": 1420016227424, "create_time": null } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/examples/test_case_data/localmeta/project/default.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/project/default.json b/examples/test_case_data/localmeta/project/default.json index 853a10a..639f3be 100644 --- a/examples/test_case_data/localmeta/project/default.json +++ b/examples/test_case_data/localmeta/project/default.json @@ -26,6 +26,16 @@ "name": "test_kylin_ii", "type": "INVERTED_INDEX", "realization": "test_kylin_ii" + }, + { + "name": "test_kylin_hybrid_inner_join", + "type": "HYBRID", + "realization": "test_kylin_hybrid_inner_join" + }, + { + "name": "test_kylin_hybrid_left_join", + "type": "HYBRID", + "realization": "test_kylin_hybrid_left_join" } ] } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java index 9c46fbb..89ed059 100644 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java +++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java @@ -338,11 +338,6 @@ public class IIInstance extends RootPersistentEntity implements IRealization { } @Override - public String getModelName() { - return this.getDescriptor().getModelName(); - } - - @Override public List<TblColRef> getAllDimensions() { return getDescriptor().listAllDimensions(); } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java ---------------------------------------------------------------------- diff --git a/metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java b/metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java index cfbe6a3..73c899b 100644 --- a/metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java +++ b/metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java @@ -49,7 +49,27 @@ public class RealizationEntry { public void setRealization(String realization) { this.realization = realization; } - + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + RealizationEntry entry = (RealizationEntry) o; + + if (realization != null ? !realization.equalsIgnoreCase(entry.realization) : entry.realization != null) return false; + if (type != entry.type) return false; + + return true; + } + + @Override + public int hashCode() { + int result = type != null ? type.hashCode() : 0; + result = 31 * result + (realization != null ? realization.hashCode() : 0); + return result; + } + @Override public String toString() { return "" + type.name() + "." + realization; http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java ---------------------------------------------------------------------- diff --git a/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java b/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java index 43f5e9a..1f3837d 100644 --- a/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java +++ b/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java @@ -69,6 +69,4 @@ public interface IRealization { public long getDateRangeEnd(); - public String getModelName(); - } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java index 3402556..8872a7d 100644 --- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java +++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java @@ -49,8 +49,8 @@ public class AdjustForWeeklyMatchedRealization extends RoutingRule { if (first instanceof HybridInstance) { HybridInstance hybrid = (HybridInstance) first; - if (hybrid.getHistoryRealizationInstance() instanceof CubeInstance) - first = hybrid.getHistoryRealizationInstance(); + if (hybrid.getRealizations()[0] instanceof CubeInstance) + first = hybrid.getRealizations()[0]; } if (first instanceof CubeInstance) { http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java index 7425d62..72995d6 100644 --- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java +++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java @@ -33,9 +33,9 @@ public class RealizationPriorityRule extends RoutingRule { static Map<RealizationType, Integer> priorities = Maps.newHashMap(); static { - priorities.put(RealizationType.HYBRID, 2); priorities.put(RealizationType.CUBE, 1); - priorities.put(RealizationType.INVERTED_INDEX, 0); + priorities.put(RealizationType.HYBRID, 1); + priorities.put(RealizationType.INVERTED_INDEX, 2); } public static void setPriorities(Map<RealizationType, Integer> priorities) { http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java ---------------------------------------------------------------------- diff --git a/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java index b58e439..864e3c1 100644 --- a/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java +++ b/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java @@ -39,9 +39,9 @@ public class IIQueryTest extends KylinQueryTest { KylinQueryTest.setUp();//invoke super class Map<RealizationType, Integer> priorities = Maps.newHashMap(); - priorities.put(RealizationType.INVERTED_INDEX, 0); - priorities.put(RealizationType.CUBE, 1); - priorities.put(RealizationType.HYBRID, 1); + priorities.put(RealizationType.INVERTED_INDEX, 1); + priorities.put(RealizationType.CUBE, 2); + priorities.put(RealizationType.HYBRID, 2); RealizationPriorityRule.setPriorities(priorities); } @@ -51,9 +51,9 @@ public class IIQueryTest extends KylinQueryTest { KylinQueryTest.tearDown();//invoke super class Map<RealizationType, Integer> priorities = Maps.newHashMap(); - priorities.put(RealizationType.INVERTED_INDEX, 1); - priorities.put(RealizationType.CUBE, 0); - priorities.put(RealizationType.HYBRID, 0); + priorities.put(RealizationType.INVERTED_INDEX, 2); + priorities.put(RealizationType.CUBE, 1); + priorities.put(RealizationType.HYBRID, 1); RealizationPriorityRule.setPriorities(priorities); } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java ---------------------------------------------------------------------- diff --git a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java index 8c3901c..415783a 100644 --- a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java +++ b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java @@ -28,6 +28,7 @@ import java.util.Properties; import org.apache.commons.lang3.StringUtils; import org.apache.kylin.query.enumerator.OLAPQuery; +import org.apache.kylin.storage.hybrid.HybridManager; import org.dbunit.database.DatabaseConnection; import org.dbunit.database.IDatabaseConnection; import org.junit.AfterClass; @@ -105,6 +106,7 @@ public class KylinQueryTest extends KylinTestBase { protected static void preferCubeOf(String joinType) { CubeManager cubeManager = CubeManager.getInstance(config); + HybridManager hybridManager = HybridManager.getInstance(config); boolean cubesBuiltInBatch = cubeManager.getCube("test_kylin_cube_with_slr_empty") != null && cubeManager.getCube("test_kylin_cube_without_slr_empty") != null && cubeManager.getCube("test_kylin_cube_with_slr_left_join_empty") != null && cubeManager.getCube("test_kylin_cube_without_slr_left_join_empty") != null; @@ -114,11 +116,15 @@ public class KylinQueryTest extends KylinTestBase { } if (joinType.equals("inner")) { + hybridManager.getHybridInstance("test_kylin_hybrid_inner_join").setCost(10); + hybridManager.getHybridInstance("test_kylin_hybrid_left_join").setCost(100); cubeManager.getCube("test_kylin_cube_with_slr_empty").setCost(20); cubeManager.getCube("test_kylin_cube_without_slr_empty").setCost(10); cubeManager.getCube("test_kylin_cube_with_slr_left_join_empty").setCost(100); cubeManager.getCube("test_kylin_cube_without_slr_left_join_empty").setCost(90); } else if (joinType.equals("left") || joinType.equals("default")) { + hybridManager.getHybridInstance("test_kylin_hybrid_left_join").setCost(20); + hybridManager.getHybridInstance("test_kylin_hybrid_inner_join").setCost(100); cubeManager.getCube("test_kylin_cube_with_slr_empty").setCost(100); cubeManager.getCube("test_kylin_cube_without_slr_empty").setCost(90); cubeManager.getCube("test_kylin_cube_with_slr_left_join_empty").setCost(20); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/server/src/main/java/org/apache/kylin/rest/service/BasicService.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java index 764dd7b..d124cf0 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java @@ -45,6 +45,7 @@ import org.apache.kylin.query.schema.OLAPSchemaFactory; import org.apache.kylin.rest.controller.QueryController; import org.apache.calcite.jdbc.Driver; import org.apache.commons.lang3.StringUtils; +import org.apache.kylin.storage.hybrid.HybridManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.cache.annotation.CacheEvict; @@ -174,6 +175,10 @@ public abstract class BasicService { return IIManager.getInstance(getConfig()); } + public final HybridManager getHybridManager() { + return HybridManager.getInstance(getConfig()); + } + protected List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, final Map<String, Output> allOutputs) { List<CubingJob> results = Lists.newArrayList(FluentIterable.from(getExecutableManager().getAllExecutables()).filter(new Predicate<AbstractExecutable>() { @Override http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/server/src/main/java/org/apache/kylin/rest/service/CacheService.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java index 9ec1621..f68fd91 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java @@ -28,6 +28,7 @@ import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.metadata.project.ProjectManager; import org.apache.kylin.metadata.realization.RealizationRegistry; import org.apache.kylin.metadata.realization.RealizationType; +import org.apache.kylin.storage.hybrid.HybridManager; import org.springframework.stereotype.Component; import java.io.IOException; @@ -45,6 +46,7 @@ public class CacheService extends BasicService { switch (cacheType) { case CUBE: getCubeManager().loadCubeCache(cacheKey); + getHybridManager().reloadHybridInstanceByChild(RealizationType.CUBE, cacheKey); cleanProjectCacheByRealization(RealizationType.CUBE, cacheKey); break; case CUBE_DESC: @@ -55,6 +57,7 @@ public class CacheService extends BasicService { break; case INVERTED_INDEX: getIIManager().loadIICache(cacheKey); + getHybridManager().reloadHybridInstanceByChild(RealizationType.INVERTED_INDEX, cacheKey); cleanProjectCacheByRealization(RealizationType.INVERTED_INDEX, cacheKey); break; case INVERTED_INDEX_DESC: @@ -76,6 +79,7 @@ public class CacheService extends BasicService { CubeManager.clearCache(); IIDescManager.clearCache(); IIManager.clearCache(); + HybridManager.clearCache(); RealizationRegistry.clearCache(); ProjectManager.clearCache(); BasicService.resetOLAPDataSources(); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/server/src/main/java/org/apache/kylin/rest/service/CubeService.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java index 71e9f19..2c7c628 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -207,9 +207,9 @@ public class CubeService extends BasicService { if (project == null) { return false; } - for (RealizationEntry projectDataModel : project.getRealizationEntries()) { - if (projectDataModel.getType() == RealizationType.CUBE) { - CubeInstance cube = getCubeManager().getCube(projectDataModel.getRealization()); + for (RealizationEntry projectRealization : project.getRealizationEntries()) { + if (projectRealization != null && projectRealization.getType() == RealizationType.CUBE) { + CubeInstance cube = getCubeManager().getCube(projectRealization.getRealization()); assert cube != null; if (cube.equals(target)) { return true; http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java index 1694719..68984cd 100644 --- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java +++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java @@ -3,11 +3,10 @@ package org.apache.kylin.storage.hybrid; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.Lists; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.RootPersistentEntity; -import org.apache.kylin.cube.CubeInstance; -import org.apache.kylin.invertedindex.IIInstance; -import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.project.RealizationEntry; @@ -16,6 +15,7 @@ import org.apache.kylin.metadata.realization.RealizationRegistry; import org.apache.kylin.metadata.realization.RealizationType; import org.apache.kylin.metadata.realization.SQLDigest; +import java.util.LinkedHashSet; import java.util.List; /** @@ -30,40 +30,90 @@ public class HybridInstance extends RootPersistentEntity implements IRealization @JsonProperty("name") private String name; - @JsonProperty("historyRealization") - private RealizationEntry historyRealization; + @JsonProperty("realizations") + private List<RealizationEntry> realizationEntries; - @JsonProperty("realTimeRealization") - private RealizationEntry realTimeRealization; + @JsonProperty("cost") + private int cost = 50; - private IRealization historyRealizationInstance; - private IRealization realTimeRealizationInstance; + private IRealization[] realizations = null; + private List<TblColRef> allDimensions = null; + private List<TblColRef> allColumns = null; + private List<MeasureDesc> allMeasures = null; + private long dateRangeStart; + private long dateRangeEnd; + private boolean isReady = false; private String projectName; - public void init() { - RealizationRegistry registry = RealizationRegistry.getInstance(config); - historyRealizationInstance = registry.getRealization(historyRealization.getType(), historyRealization.getRealization()); - realTimeRealizationInstance = registry.getRealization(realTimeRealization.getType(), realTimeRealization.getRealization()); + private boolean initiated = false; - if (historyRealizationInstance == null) { - throw new IllegalArgumentException("Didn't find realization '" + historyRealization.getType() + "'" + " with name '" + historyRealization.getRealization() + "' in '" + name + "'"); - } + public List<RealizationEntry> getRealizationEntries() { + return realizationEntries; + } - if (realTimeRealizationInstance == null) { - throw new IllegalArgumentException("Didn't find realization '" + realTimeRealization.getType() + "'" + " with name '" + realTimeRealization.getRealization() + "' in '" + name + "'"); - } + private void init() { + if (initiated == true) + return; + + synchronized (this) { + if (initiated == true) + return; + + if (realizationEntries == null || realizationEntries.size() == 0) + throw new IllegalArgumentException(); + + RealizationRegistry registry = RealizationRegistry.getInstance(config); + realizations = new IRealization[realizationEntries.size()]; + for (int i = 0; i < realizationEntries.size(); i++) { + IRealization realization = registry.getRealization(realizationEntries.get(i).getType(), realizationEntries.get(i).getRealization()); + if (realization == null) + throw new IllegalArgumentException("Realization '" + realizationEntries.get(i) + "' not loaded."); + realizations[i] = realization; + } + + LinkedHashSet<TblColRef> columns = new LinkedHashSet<TblColRef>(); + LinkedHashSet<TblColRef> dimensions = new LinkedHashSet<TblColRef>(); + LinkedHashSet<MeasureDesc> measures = new LinkedHashSet<MeasureDesc>(); + dateRangeStart = 0; + dateRangeEnd = Long.MAX_VALUE; + for (IRealization realization : realizations) { + if (realization.isReady() == false) + continue; + + columns.addAll(realization.getAllColumns()); + dimensions.addAll(realization.getAllDimensions()); + measures.addAll(realization.getMeasures()); + + if (realization.isReady()) + isReady = true; + if (dateRangeStart == 0 || realization.getDateRangeStart() < dateRangeStart) + dateRangeStart = realization.getDateRangeStart(); + + if (dateRangeStart == Long.MAX_VALUE || realization.getDateRangeEnd() > dateRangeEnd) + dateRangeEnd = realization.getDateRangeEnd(); + } + + allDimensions = Lists.newArrayList(dimensions); + allColumns = Lists.newArrayList(columns); + allMeasures = Lists.newArrayList(measures); + + initiated = true; + } } @Override public boolean isCapable(SQLDigest digest) { - return getHistoryRealizationInstance().isCapable(digest) && getRealTimeRealizationInstance().isCapable(digest); + for (IRealization realization : getRealizations()) { + if (realization.isCapable(digest)) + return true; + } + return false; } @Override public int getCost(SQLDigest digest) { - return historyRealizationInstance.getCost(digest); - //return Math.min(historyRealizationInstance.getCost(digest), realTimeRealizationInstance.getCost(digest)); + return cost; } @Override @@ -73,22 +123,24 @@ public class HybridInstance extends RootPersistentEntity implements IRealization @Override public String getFactTable() { - return getHistoryRealizationInstance().getFactTable(); + return getRealizations()[0].getFactTable(); } @Override public List<TblColRef> getAllColumns() { - return getHistoryRealizationInstance().getAllColumns(); + init(); + return allColumns; } @Override public List<MeasureDesc> getMeasures() { - return getHistoryRealizationInstance().getMeasures(); + init(); + return allMeasures; } @Override public boolean isReady() { - return historyRealizationInstance.isReady() || realTimeRealizationInstance.isReady(); + return isReady; } @Override @@ -128,61 +180,32 @@ public class HybridInstance extends RootPersistentEntity implements IRealization this.config = config; } - public RealizationEntry getHistoryRealization() { - return historyRealization; - } - - public RealizationEntry getRealTimeRealization() { - return realTimeRealization; - } - - public IRealization getHistoryRealizationInstance() { - if (historyRealizationInstance == null) { - this.init(); - } - return historyRealizationInstance; - } - - public IRealization getRealTimeRealizationInstance() { - if (realTimeRealizationInstance == null) { - this.init(); - } - return realTimeRealizationInstance; - } - @Override public long getDateRangeStart() { - return Math.min(getHistoryRealizationInstance().getDateRangeStart(), getRealTimeRealizationInstance().getDateRangeStart()); + return dateRangeStart; } @Override public long getDateRangeEnd() { - return Math.max(getHistoryRealizationInstance().getDateRangeEnd(), getRealTimeRealizationInstance().getDateRangeEnd()) +1; + return dateRangeEnd; } - - - public DataModelDesc getDataModelDesc(){ - if (getHistoryRealizationInstance() instanceof CubeInstance) { - return ((CubeInstance) historyRealizationInstance).getDescriptor().getModel(); - } - - return ((IIInstance) historyRealizationInstance).getDescriptor().getModel(); + @Override + public List<TblColRef> getAllDimensions() { + init(); + return allDimensions; } - @Override - public String getModelName() { - if (getHistoryRealizationInstance() instanceof CubeInstance) { - return ((CubeInstance) historyRealizationInstance).getDescriptor().getModelName(); - } - - return ((IIInstance) historyRealizationInstance).getDescriptor().getModelName(); + public IRealization[] getRealizations() { + init(); + return realizations; } - @Override - public List<TblColRef> getAllDimensions(){ - - return this.getHistoryRealizationInstance().getAllDimensions(); + public static String concatResourcePath(String hybridName) { + return ResourceStore.HYBRID_RESOURCE_ROOT + "/" + hybridName + ".json"; } + public void setCost(int cost) { + this.cost = cost; + } } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java index df3fae3..8373cf1 100644 --- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java +++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java @@ -81,6 +81,21 @@ public class HybridManager implements IRealizationProvider { logger.debug("Loaded " + paths.size() + " Hybrid(s)"); } + public void reloadHybridInstanceByChild(RealizationType type, String realizationName) throws IOException { + for (HybridInstance hybridInstance : hybridMap.values()) { + boolean includes = false; + for (IRealization realization : hybridInstance.getRealizations()) { + if (realization.getType() == type && realization.getName().equalsIgnoreCase(realizationName)) { + includes = true; + break; + } + } + + if (includes == true) + loadHybridInstance(HybridInstance.concatResourcePath(hybridInstance.getName())); + } + } + private synchronized HybridInstance loadHybridInstance(String path) throws IOException { ResourceStore store = getStore(); @@ -89,14 +104,13 @@ public class HybridManager implements IRealizationProvider { hybridInstance = store.getResource(path, HybridInstance.class, HYBRID_SERIALIZER); hybridInstance.setConfig(config); + if (hybridInstance.getRealizationEntries() == null || hybridInstance.getRealizationEntries().size() == 0) { + throw new IllegalStateException("HybridInstance must have realization entries, " + path); + } + if (StringUtils.isBlank(hybridInstance.getName())) throw new IllegalStateException("HybridInstance name must not be blank, at " + path); - if (hybridInstance.getHistoryRealization() == null || hybridInstance.getRealTimeRealization() == null) { - - throw new IllegalStateException("HybridInstance must have both historyRealization and realTimeRealization set, at " + path); - } - final String name = hybridInstance.getName(); hybridMap.putLocal(name, hybridInstance); @@ -124,4 +138,5 @@ public class HybridManager implements IRealizationProvider { private ResourceStore getStore() { return ResourceStore.getStore(this.config); } + } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java index 7ad5ed0..11ad211 100644 --- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java +++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java @@ -1,69 +1,43 @@ package org.apache.kylin.storage.hybrid; -import com.google.common.base.Function; import com.google.common.collect.Lists; -import com.google.common.collect.Ranges; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.metadata.MetadataManager; -import org.apache.kylin.metadata.model.DataModelDesc; -import org.apache.kylin.metadata.model.TblColRef; +import org.apache.kylin.metadata.realization.IRealization; import org.apache.kylin.metadata.realization.SQLDigest; -import org.apache.kylin.metadata.realization.SQLDigestUtil; import org.apache.kylin.metadata.tuple.CompoundTupleIterator; import org.apache.kylin.metadata.tuple.ITupleIterator; import org.apache.kylin.storage.IStorageEngine; import org.apache.kylin.storage.StorageContext; import org.apache.kylin.storage.StorageEngineFactory; -import org.apache.kylin.storage.tuple.TupleInfo; -import javax.annotation.Nullable; +import java.util.List; /** */ public class HybridStorageEngine implements IStorageEngine { - private HybridInstance hybridInstance; - private IStorageEngine historicalStorageEngine; - private IStorageEngine realtimeStorageEngine; + private IRealization[] realizations; + private IStorageEngine[] storageEngines; public HybridStorageEngine(HybridInstance hybridInstance) { - this.hybridInstance = hybridInstance; - this.historicalStorageEngine = StorageEngineFactory.getStorageEngine(this.hybridInstance.getHistoryRealizationInstance()); - this.realtimeStorageEngine = StorageEngineFactory.getStorageEngine(this.hybridInstance.getRealTimeRealizationInstance()); + this.realizations = hybridInstance.getRealizations(); + storageEngines = new IStorageEngine[realizations.length]; + for (int i = 0; i < realizations.length; i++) { + storageEngines[i] = StorageEngineFactory.getStorageEngine(realizations[i]); + } } @Override public ITupleIterator search(final StorageContext context, final SQLDigest sqlDigest) { - - // search the historic realization - ITupleIterator historicalDataIterator = this.historicalStorageEngine.search(context, sqlDigest); - - String modelName = hybridInstance.getModelName(); - MetadataManager metaMgr = getMetadataManager(); - DataModelDesc modelDesc = metaMgr.getDataModelDesc(modelName); - - // if the model isn't partitioned, only query the history - if (modelDesc.getPartitionDesc() == null || modelDesc.getPartitionDesc().getPartitionDateColumnRef() == null) - return historicalDataIterator; - - TblColRef partitionColRef = modelDesc.getPartitionDesc().getPartitionDateColumnRef(); - - ITupleIterator realtimeDataIterator = SQLDigestUtil.appendTsFilterToExecute(sqlDigest, partitionColRef, // - Ranges.atLeast(hybridInstance.getHistoryRealizationInstance().getDateRangeEnd()),// - new Function<Void, ITupleIterator>() { - @Nullable - @Override - public ITupleIterator apply(@Nullable Void input) { - return realtimeStorageEngine.search(context, sqlDigest); - } - }); - - // combine history and real-time tuple iterator - return new CompoundTupleIterator(Lists.newArrayList(historicalDataIterator, realtimeDataIterator)); + List<ITupleIterator> tupleIterators = Lists.newArrayList(); + for (int i = 0; i < realizations.length; i++) { + if (realizations[i].isReady() && realizations[i].isCapable(sqlDigest)) { + ITupleIterator dataIterator = storageEngines[i].search(context, sqlDigest); + tupleIterators.add(dataIterator); + } + } + // combine tuple iterator + return new CompoundTupleIterator(tupleIterators); } - private MetadataManager getMetadataManager() { - return MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); - } } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java ---------------------------------------------------------------------- diff --git a/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java b/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java index 3ad7ca6..8a1bcf0 100644 --- a/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java +++ b/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java @@ -26,15 +26,12 @@ public class HybridManagerTest extends LocalFileMetadataTestCase { @Test public void testBasics() throws Exception { - HybridInstance cube = getHybridManager().getHybridInstance("test_kylin_hybrid_left_join"); - cube.init(); - System.out.println(JsonUtil.writeValueAsIndentString(cube)); + HybridInstance hybridInstance = getHybridManager().getHybridInstance("test_kylin_hybrid_left_join"); + System.out.println(JsonUtil.writeValueAsIndentString(hybridInstance)); - IRealization history = cube.getHistoryRealizationInstance(); - IRealization realTime = cube.getRealTimeRealizationInstance(); + IRealization history = hybridInstance.getRealizations()[0]; Assert.assertTrue(history instanceof CubeInstance); - Assert.assertTrue(realTime instanceof IIInstance); }
