KYLIN-867 a bug fix in LookupTableEnumerator
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ab3b07df Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ab3b07df Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ab3b07df Branch: refs/heads/0.7-staging Commit: ab3b07dfb77f40f99cf5cbb143345464772bd586 Parents: 4f742cd Author: shaofengshi <[email protected]> Authored: Sun Jul 5 13:53:33 2015 +0800 Committer: shaofengshi <[email protected]> Committed: Mon Jul 6 16:19:17 2015 +0800 ---------------------------------------------------------------------- .../query/enumerator/LookupTableEnumerator.java | 19 ++++++- .../kylin/storage/hybrid/HybridInstance.java | 55 ++++++++++++++++---- 2 files changed, 63 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab3b07df/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java index 3ae19dd..733e8fe 100644 --- a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java +++ b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java @@ -29,9 +29,13 @@ import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.model.DimensionDesc; import org.apache.kylin.dict.lookup.LookupStringTable; import org.apache.kylin.metadata.model.ColumnDesc; +import org.apache.kylin.metadata.realization.IRealization; import org.apache.kylin.query.relnode.OLAPContext; import org.apache.kylin.query.schema.OLAPTable; +import org.apache.kylin.storage.hybrid.HybridInstance; import org.apache.kylin.storage.tuple.Tuple; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** */ @@ -42,10 +46,23 @@ public class LookupTableEnumerator implements Enumerator<Object[]> { private final Object[] current; private Iterator<String[]> iterator; + private final static Logger logger = LoggerFactory.getLogger(LookupTableEnumerator.class); public LookupTableEnumerator(OLAPContext olapContext) { //TODO: assuming LookupTableEnumerator is handled by a cube - CubeInstance cube = (CubeInstance) olapContext.realization; + CubeInstance cube = null; + + if (olapContext.realization instanceof CubeInstance) + cube = (CubeInstance) olapContext.realization; + else if (olapContext.realization instanceof HybridInstance) { + final HybridInstance hybridInstance = (HybridInstance)olapContext.realization; + final IRealization latestRealization = hybridInstance.getLatestRealization(); + if (latestRealization instanceof CubeInstance) { + cube = (CubeInstance) latestRealization; + } else { + throw new IllegalStateException(); + } + } String lookupTableName = olapContext.firstTableScan.getTableName(); DimensionDesc dim = cube.getDescriptor().findDimensionByTable(lookupTableName); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab3b07df/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java index 68984cd..e7071f9 100644 --- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java +++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java @@ -14,9 +14,10 @@ import org.apache.kylin.metadata.realization.IRealization; import org.apache.kylin.metadata.realization.RealizationRegistry; import org.apache.kylin.metadata.realization.RealizationType; import org.apache.kylin.metadata.realization.SQLDigest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.util.LinkedHashSet; -import java.util.List; +import java.util.*; /** */ @@ -46,6 +47,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization private String projectName; private boolean initiated = false; + private final static Logger logger = LoggerFactory.getLogger(HybridInstance.class); public List<RealizationEntry> getRealizationEntries() { return realizationEntries; @@ -63,12 +65,18 @@ public class HybridInstance extends RootPersistentEntity implements IRealization throw new IllegalArgumentException(); RealizationRegistry registry = RealizationRegistry.getInstance(config); - realizations = new IRealization[realizationEntries.size()]; + List<IRealization> realizationList = Lists.newArrayList(); for (int i = 0; i < realizationEntries.size(); i++) { IRealization realization = registry.getRealization(realizationEntries.get(i).getType(), realizationEntries.get(i).getRealization()); - if (realization == null) - throw new IllegalArgumentException("Realization '" + realizationEntries.get(i) + "' not loaded."); - realizations[i] = realization; + if (realization == null) { + logger.error("Realization '" + realization.getName() + " is not found, remove from Hybrid '" + this.getName() + "'"); + continue; + } + if (realization.isReady() == false) { + logger.error("Realization '" + realization.getName() + " is disabled, remove from Hybrid '" + this.getName() + "'"); + continue; + } + realizationList.add(realization); } LinkedHashSet<TblColRef> columns = new LinkedHashSet<TblColRef>(); @@ -76,10 +84,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization LinkedHashSet<MeasureDesc> measures = new LinkedHashSet<MeasureDesc>(); dateRangeStart = 0; dateRangeEnd = Long.MAX_VALUE; - for (IRealization realization : realizations) { - if (realization.isReady() == false) - continue; - + for (IRealization realization : realizationList) { columns.addAll(realization.getAllColumns()); dimensions.addAll(realization.getAllDimensions()); measures.addAll(realization.getMeasures()); @@ -98,6 +103,29 @@ public class HybridInstance extends RootPersistentEntity implements IRealization allColumns = Lists.newArrayList(columns); allMeasures = Lists.newArrayList(measures); + Collections.sort(realizationList, new Comparator<IRealization>() { + @Override + public int compare(IRealization o1, IRealization o2) { + + long i1 = o1.getDateRangeStart(); + long i2 = o2.getDateRangeStart(); + long comp = i1 - i2; + if (comp != 0) { + return comp > 0 ? 1 : -1; + } + + i1 = o1.getDateRangeEnd(); + i2 = o2.getDateRangeEnd(); + comp = i1 - i2; + if (comp != 0) { + return comp > 0 ? 1 : -1; + } + + return 0; + } + }); + + this.realizations = realizationList.toArray(new IRealization[realizationList.size()]); initiated = true; } } @@ -208,4 +236,11 @@ public class HybridInstance extends RootPersistentEntity implements IRealization public void setCost(int cost) { this.cost = cost; } + + public IRealization getLatestRealization() { + if (realizations.length > 0) { + return realizations[realizations.length - 1]; + } + return null; + } }
