KYLIN-2783 Code review clean up This closes #69
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b3b7a4e7 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b3b7a4e7 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b3b7a4e7 Branch: refs/heads/ranger Commit: b3b7a4e7e04831924ff1f6d844956ca5df9dcab1 Parents: 0eded0a Author: Li Yang <liy...@apache.org> Authored: Sat Sep 9 21:09:02 2017 +0800 Committer: Li Yang <liy...@apache.org> Committed: Sun Sep 10 07:47:20 2017 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/cube/CubeInstance.java | 47 +++----------------- .../java/org/apache/kylin/cube/CubeManager.java | 6 +-- .../org/apache/kylin/cube/cuboid/Cuboid.java | 37 +++++++-------- .../org/apache/kylin/cube/cuboid/CuboidCLI.java | 14 +++--- .../kylin/cube/cuboid/CuboidScheduler.java | 8 ++-- .../cube/cuboid/DefaultCuboidScheduler.java | 8 ++-- .../kylin/cube/cuboid/TreeCuboidScheduler.java | 2 +- .../inmemcubing/AbstractInMemCubeBuilder.java | 5 --- .../kylin/cube/model/AggregationGroup.java | 2 +- .../org/apache/kylin/cube/model/CubeDesc.java | 41 +++++++++-------- .../apache/kylin/cube/cuboid/CuboidTest.java | 2 - .../apache/kylin/engine/spark/SparkCubing.java | 4 +- 12 files changed, 67 insertions(+), 109 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/b3b7a4e7/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java index fa270fd..d5d6ac9 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java @@ -18,9 +18,6 @@ package org.apache.kylin.cube; -import static org.apache.kylin.cube.cuboid.CuboidModeEnum.CURRENT; -import static org.apache.kylin.cube.cuboid.CuboidModeEnum.RECOMMEND; - import java.io.IOException; import java.util.HashSet; import java.util.List; @@ -33,7 +30,6 @@ import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.RootPersistentEntity; import org.apache.kylin.common.util.CompressionUtils; import org.apache.kylin.common.util.JsonUtil; -import org.apache.kylin.cube.cuboid.CuboidModeEnum; import org.apache.kylin.cube.cuboid.CuboidScheduler; import org.apache.kylin.cube.cuboid.TreeCuboidScheduler; import org.apache.kylin.cube.model.CubeDesc; @@ -106,14 +102,14 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, @JsonProperty("create_time_utc") private long createTimeUTC; - @JsonProperty("cuboidBytes") + @JsonProperty("cuboid_bytes") private byte[] cuboidBytes; @JsonProperty("cuboid_bytes_recommend") private byte[] cuboidBytesRecommend; - @JsonProperty("last_optimized") - private long lastOptimized; + @JsonProperty("cuboid_last_optimized") + private long cuboidLastOptimized; // cuboid scheduler lazy built transient private CuboidScheduler cuboidScheduler; @@ -332,36 +328,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, this.createTimeUTC = createTimeUTC; } - public Set<Long> getCuboidsByMode(String cuboidModeName) { - return getCuboidsByMode(cuboidModeName == null ? null : CuboidModeEnum.getByModeName(cuboidModeName)); - } - - public Set<Long> getCuboidsByMode(CuboidModeEnum cuboidMode) { - if (cuboidMode == null || cuboidMode == CURRENT) { - return getCuboidScheduler().getAllCuboidIds(); - } - Set<Long> cuboidsRecommend = getCuboidsRecommend(); - if (cuboidsRecommend == null || cuboidMode == RECOMMEND) { - return cuboidsRecommend; - } - Set<Long> currentCuboids = getCuboidScheduler().getAllCuboidIds(); - switch (cuboidMode) { - case RECOMMEND_EXISTING: - cuboidsRecommend.retainAll(currentCuboids); - return cuboidsRecommend; - case RECOMMEND_MISSING: - cuboidsRecommend.removeAll(currentCuboids); - return cuboidsRecommend; - case RECOMMEND_MISSING_WITH_BASE: - cuboidsRecommend.removeAll(currentCuboids); - currentCuboids.add(getCuboidScheduler().getBaseCuboidId()); - return cuboidsRecommend; - default: - return null; - } - } - - public Map<Long, Long> getCuboids() { + Map<Long, Long> getCuboids() { if (cuboidBytes == null) return null; byte[] uncompressed; @@ -377,7 +344,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, } } - public void setCuboids(Map<Long, Long> cuboids) { + void setCuboids(Map<Long, Long> cuboids) { if (cuboids == null) return; if (cuboids.isEmpty()) { @@ -394,7 +361,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, } } - public Set<Long> getCuboidsRecommend() { + Set<Long> getCuboidsRecommend() { if (cuboidBytesRecommend == null) return null; byte[] uncompressed; @@ -410,7 +377,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, } } - public void setCuboidsRecommend(HashSet<Long> cuboids) { + void setCuboidsRecommend(HashSet<Long> cuboids) { if (cuboids == null) return; if (cuboids.isEmpty()) { http://git-wip-us.apache.org/repos/asf/kylin/blob/b3b7a4e7/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index 6ebfd99..eb330ae 100755 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -310,7 +310,7 @@ public class CubeManager implements IRealizationProvider { // remove cube and update cache getStore().deleteResource(cube.getResourcePath()); cubeMap.remove(cube.getName()); - Cuboid.reloadCache(cube); + Cuboid.clearCache(cube); if (deleteDesc && cube.getDescriptor() != null) { CubeDescManager.getInstance(config).removeCubeDesc(cube.getDescriptor()); @@ -632,7 +632,7 @@ public class CubeManager implements IRealizationProvider { */ public CubeInstance reloadCubeLocal(String cubeName) { CubeInstance cubeInstance = reloadCubeLocalAt(CubeInstance.concatResourcePath(cubeName)); - Cuboid.reloadCache(cubeInstance); + Cuboid.clearCache(cubeInstance); return cubeInstance; } @@ -643,7 +643,7 @@ public class CubeManager implements IRealizationProvider { for (CubeSegment segment : cube.getSegments()) { usedStorageLocation.remove(segment.getUuid()); } - Cuboid.reloadCache(cube); + Cuboid.clearCache(cube); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/b3b7a4e7/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java index fa83775..f314baf 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java @@ -45,7 +45,7 @@ import com.google.common.collect.Maps; @SuppressWarnings("serial") public class Cuboid implements Comparable<Cuboid>, Serializable { - //TODO Guava cache may be better + // TODO Should the cache be inside CuboidScheduler? private final static Map<String, Map<Long, Cuboid>> CUBOID_CACHE = Maps.newConcurrentMap(); // smaller is better @@ -88,11 +88,6 @@ public class Cuboid implements Comparable<Cuboid>, Serializable { return cuboidID; } - // for mandatory cuboid, no need to translate cuboid - public static Cuboid findForMandatory(CubeDesc cube, long cuboidID) { - return new Cuboid(cube, cuboidID, cuboidID); - } - public static Cuboid findById(CuboidScheduler cuboidScheduler, byte[] cuboidID) { return findById(cuboidScheduler, Bytes.toLong(cuboidID)); } @@ -111,10 +106,10 @@ public class Cuboid implements Comparable<Cuboid>, Serializable { } public static Cuboid findById(CuboidScheduler cuboidScheduler, long cuboidID) { - Map<Long, Cuboid> cubeCache = CUBOID_CACHE.get(cuboidScheduler.getResponsibleKey()); + Map<Long, Cuboid> cubeCache = CUBOID_CACHE.get(cuboidScheduler.getCuboidCacheKey()); if (cubeCache == null) { cubeCache = Maps.newConcurrentMap(); - CUBOID_CACHE.put(cuboidScheduler.getResponsibleKey(), cubeCache); + CUBOID_CACHE.put(cuboidScheduler.getCuboidCacheKey(), cubeCache); } Cuboid cuboid = cubeCache.get(cuboidID); if (cuboid == null) { @@ -130,7 +125,19 @@ public class Cuboid implements Comparable<Cuboid>, Serializable { } public static Cuboid getBaseCuboid(CubeDesc cube) { - return findForMandatory(cube, getBaseCuboidId(cube)); + return findById(cube.getInitialCuboidScheduler(), getBaseCuboidId(cube)); + } + + public static void clearCache() { + CUBOID_CACHE.clear(); + } + + public static void clearCache(CubeInstance cubeInstance) { + clearCache(cubeInstance.getCuboidScheduler()); + } + + private static void clearCache(CuboidScheduler cuboidScheduler) { + CUBOID_CACHE.remove(cuboidScheduler.getCuboidCacheKey()); } // ============================================================================ @@ -226,18 +233,6 @@ public class Cuboid implements Comparable<Cuboid>, Serializable { return requirePostAggregation; } - public static void clearCache() { - CUBOID_CACHE.clear(); - } - - public static void reloadCache(CubeInstance cubeInstance) { - reloadCache(cubeInstance.getCuboidScheduler()); - } - - private static void reloadCache(CuboidScheduler cuboidScheduler) { - CUBOID_CACHE.remove(cuboidScheduler.getResponsibleKey()); - } - @Override public String toString() { return "Cuboid [id=" + id + "]"; http://git-wip-us.apache.org/repos/asf/kylin/blob/b3b7a4e7/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java index 4300a6c..fa8e879 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java @@ -88,28 +88,28 @@ public class CuboidCLI { //check all valid and invalid for (long i = 0; i < baseCuboid; ++i) { if (cuboidSet.contains(i)) { - if (!cubeDesc.getInitialCuboidScheduler().isValid(i)) { + if (!scheduler.isValid(i)) { throw new RuntimeException(); } - if (cubeDesc.getInitialCuboidScheduler().findBestMatchCuboid(i) != i) { + if (scheduler.findBestMatchCuboid(i) != i) { throw new RuntimeException(); } } else { - if (cubeDesc.getInitialCuboidScheduler().isValid(i)) { + if (scheduler.isValid(i)) { throw new RuntimeException(); } - long corrected = cubeDesc.getInitialCuboidScheduler().findBestMatchCuboid(i); + long corrected = scheduler.findBestMatchCuboid(i); if (corrected == i) { throw new RuntimeException(); } - if (!cubeDesc.getInitialCuboidScheduler().isValid(corrected)) { + if (!scheduler.isValid(corrected)) { throw new RuntimeException(); } - if (cubeDesc.getInitialCuboidScheduler().findBestMatchCuboid(corrected) != corrected) { + if (scheduler.findBestMatchCuboid(corrected) != corrected) { throw new RuntimeException(); } } @@ -133,7 +133,7 @@ public class CuboidCLI { } public static int[] calculateAllLevelCount(CubeDesc cube) { - int levels = cube.getInitialBuildLevel(); + int levels = cube.getInitialCuboidScheduler().getBuildLevel(); int[] allLevelCounts = new int[levels + 1]; CuboidScheduler scheduler = cube.getInitialCuboidScheduler(); http://git-wip-us.apache.org/repos/asf/kylin/blob/b3b7a4e7/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java index cef0f77..5de7e9e 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java @@ -60,14 +60,14 @@ abstract public class CuboidScheduler { /** Returns the child cuboids of a parent. */ abstract public List<Long> getSpanningCuboid(long parentCuboid); - /** Returns a cuboid on the tree that best matches the request cuboid. */ + /** Returns a valid cuboid that best matches the request cuboid. */ abstract public long findBestMatchCuboid(long requestCuboid); - /** Returns whether requestCuboid is valid or not*/ + /** Checks whether a cuboid is valid or not. */ abstract public boolean isValid(long requestCuboid); - /** Returns the key for what this cuboid scheduler responsible for*/ - abstract public String getResponsibleKey(); + /** Returns the key for what this cuboid scheduler responsible for. */ + abstract public String getCuboidCacheKey(); // ============================================================================ http://git-wip-us.apache.org/repos/asf/kylin/blob/b3b7a4e7/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java index cba417a..9762cdc 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java @@ -102,7 +102,7 @@ public class DefaultCuboidScheduler extends CuboidScheduler { return findBestMatchCuboid1(cuboid); } - public long findBestMatchCuboid1(long cuboid) { + long findBestMatchCuboid1(long cuboid) { if (isValid(cuboid)) { return cuboid; } @@ -127,7 +127,7 @@ public class DefaultCuboidScheduler extends CuboidScheduler { return doFindBestMatchCuboid1(onTreeCandi); } - public long doFindBestMatchCuboid1(long cuboid) { + private long doFindBestMatchCuboid1(long cuboid) { long parent = getOnTreeParent(cuboid); while (parent > 0) { if (cubeDesc.getAllCuboids().contains(parent)) { @@ -466,7 +466,7 @@ public class DefaultCuboidScheduler extends CuboidScheduler { return Long.bitCount(cuboidID) <= dimCap; } - public long findBestMatchCuboid2(long cuboid) { + long findBestMatchCuboid2(long cuboid) { long bestParent = doFindBestMatchCuboid2(cuboid, Cuboid.getBaseCuboidId(cubeDesc)); if (bestParent < -1) { throw new IllegalStateException("Cannot find the parent of the cuboid:" + cuboid); @@ -499,7 +499,7 @@ public class DefaultCuboidScheduler extends CuboidScheduler { return (cuboidId & ~parentCuboid) == 0; } - public String getResponsibleKey() { + public String getCuboidCacheKey() { return CubeDesc.class.getName() + "-" + cubeDesc.getName(); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/b3b7a4e7/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java index 414fc4a..e0e82aa 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java @@ -316,7 +316,7 @@ public class TreeCuboidScheduler extends CuboidScheduler { } } - public String getResponsibleKey() { + public String getCuboidCacheKey() { return CubeInstance.class.getName() + "-" + cubeDesc.getName(); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/b3b7a4e7/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java index ae38261..952926c 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java @@ -51,11 +51,6 @@ abstract public class AbstractInMemCubeBuilder { protected int taskThreadCount = 1; protected int reserveMemoryMB = 100; - // @Deprecated - // public AbstractInMemCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { - // this(cubeDesc.getInitialCuboidScheduler(), cubeDesc, flatDesc, dictionaryMap); - // } - protected AbstractInMemCubeBuilder(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { if (cuboidScheduler == null) http://git-wip-us.apache.org/repos/asf/kylin/blob/b3b7a4e7/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java index 852451d..c1dbc1e 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java @@ -296,7 +296,7 @@ public class AggregationGroup implements Serializable { long combination = 1; if (this.getDimCap() > 0) { - DefaultCuboidScheduler cuboidScheduler = cubeDesc.getInitialCuboidScheduler(); + DefaultCuboidScheduler cuboidScheduler = (DefaultCuboidScheduler) cubeDesc.getInitialCuboidScheduler(); combination = cuboidScheduler.calculateCuboidsForAggGroup(this).size(); } else { Set<String> includeDims = new TreeSet<>(Arrays.asList(includes)); http://git-wip-us.apache.org/repos/asf/kylin/blob/b3b7a4e7/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index dff3485..f0b14cc 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -51,7 +51,7 @@ import org.apache.kylin.common.persistence.RootPersistentEntity; import org.apache.kylin.common.util.Array; import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.common.util.Pair; -import org.apache.kylin.cube.cuboid.DefaultCuboidScheduler; +import org.apache.kylin.cube.cuboid.CuboidScheduler; import org.apache.kylin.measure.MeasureType; import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType; import org.apache.kylin.metadata.MetadataConstants; @@ -168,15 +168,6 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { @JsonProperty("override_kylin_properties") private LinkedHashMap<String, String> overrideKylinProps = new LinkedHashMap<String, String>(); - private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<>(); - private LinkedHashSet<ColumnDesc> allColumnDescs = new LinkedHashSet<>(); - private LinkedHashSet<TblColRef> dimensionColumns = new LinkedHashSet<>(); - - private Map<TblColRef, DeriveInfo> derivedToHostMap = Maps.newHashMap(); - private Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedMap = Maps.newHashMap(); - - private Map<TblColRef, DeriveInfo> extendedColumnToHosts = Maps.newHashMap(); - @JsonProperty("partition_offset_start") @JsonInclude(JsonInclude.Include.NON_EMPTY) private Map<Integer, Long> partitionOffsetStart = Maps.newHashMap(); @@ -189,6 +180,17 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { @JsonInclude(JsonInclude.Include.NON_NULL) private int parentForward = 3; + private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<>(); + private LinkedHashSet<ColumnDesc> allColumnDescs = new LinkedHashSet<>(); + private LinkedHashSet<TblColRef> dimensionColumns = new LinkedHashSet<>(); + + private Map<TblColRef, DeriveInfo> derivedToHostMap = Maps.newHashMap(); + private Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedMap = Maps.newHashMap(); + + private Map<TblColRef, DeriveInfo> extendedColumnToHosts = Maps.newHashMap(); + + transient private CuboidScheduler cuboidScheduler = null; + public boolean isEnableSharding() { //in the future may extend to other storage that is shard-able return storageType != IStorageAware.ID_HBASE && storageType != IStorageAware.ID_HYBRID; @@ -550,6 +552,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { hostToDerivedMap = Maps.newHashMap(); extendedColumnToHosts = Maps.newHashMap(); cuboidBlackSet = Sets.newHashSet(); + cuboidScheduler = null; } public void init(KylinConfig config) { @@ -607,16 +610,16 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { amendAllColumns(); } - public DefaultCuboidScheduler getInitialCuboidScheduler() { - return new DefaultCuboidScheduler(this); - } + public CuboidScheduler getInitialCuboidScheduler() { + if (cuboidScheduler != null) + return cuboidScheduler; - /** - * Get cuboid level count except base cuboid - * @return - */ - public int getInitialBuildLevel() { - return getInitialCuboidScheduler().getCuboidsByLayer().size() - 1; + synchronized (this) { + if (cuboidScheduler == null) { + cuboidScheduler = CuboidScheduler.getInstance(this); + } + return cuboidScheduler; + } } public boolean isBlackedCuboid(long cuboidID) { http://git-wip-us.apache.org/repos/asf/kylin/blob/b3b7a4e7/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java index 8dc944b..0fa849e 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java @@ -163,7 +163,6 @@ public class CuboidTest extends LocalFileMetadataTestCase { @Test public void testFindCuboidByIdWithSingleAggrGroup2() { CubeDesc cube = getTestKylinCubeWithSeller(); - CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler(); Cuboid cuboid; cuboid = Cuboid.findById(cube, 0); @@ -191,7 +190,6 @@ public class CuboidTest extends LocalFileMetadataTestCase { @Test public void testFindCuboidByIdWithMultiAggrGroup() { CubeDesc cube = getTestKylinCubeWithoutSellerLeftJoin(); - CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler(); Cuboid cuboid; cuboid = Cuboid.findById(cube, toLong("111111110")); http://git-wip-us.apache.org/repos/asf/kylin/blob/b3b7a4e7/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java index fd53c5a..7205e1b 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java @@ -377,8 +377,8 @@ public class SparkCubing extends AbstractApplication { LinkedBlockingQueue<List<String>> blockingQueue = new LinkedBlockingQueue(); System.out.println("load properties finished"); IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(cubeSegment); - AbstractInMemCubeBuilder inMemCubeBuilder = new DoggedCubeBuilder( - cubeSegment.getCuboidScheduler(), flatDesc, dictionaryMap); + AbstractInMemCubeBuilder inMemCubeBuilder = new DoggedCubeBuilder( + cubeSegment.getCuboidScheduler(), flatDesc, dictionaryMap); final SparkCuboidWriter sparkCuboidWriter = new BufferedCuboidWriter(new DefaultTupleConverter(cubeInstance.getSegmentById(segmentId), columnLengthMap)); Executors.newCachedThreadPool().submit(inMemCubeBuilder.buildAsRunnable(blockingQueue, sparkCuboidWriter)); try {
