KYLIN-2783 merge AggregationGroupScheduler into CuboidScheduler
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c8bab3df Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c8bab3df Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c8bab3df Branch: refs/heads/master Commit: c8bab3df33edd5dc146a02f94caccd0d810b68bd Parents: 5008866 Author: Li Yang <liy...@apache.org> Authored: Sun Aug 13 16:34:22 2017 +0800 Committer: Roger Shi <rogershijich...@gmail.com> Committed: Mon Aug 14 09:35:09 2017 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/KylinVersion.java | 24 ++- .../apache/kylin/common/KylinVersionTest.java | 19 +- .../cube/cuboid/AggregationGroupScheduler.java | 126 ------------ .../kylin/cube/cuboid/CuboidScheduler.java | 4 + .../cube/cuboid/DefaultCuboidScheduler.java | 191 ++++++++++++++----- .../kylin/cube/model/AggregationGroup.java | 10 +- .../org/apache/kylin/cube/model/SelectRule.java | 6 +- 7 files changed, 187 insertions(+), 193 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/c8bab3df/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java index 960bafe..c6913fc 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java @@ -37,10 +37,11 @@ public class KylinVersion implements Comparable { private static final String COMMIT_SHA1_v15 = "commit_SHA1"; private static final String COMMIT_SHA1_v13 = "commit.sha1"; - public int major; - public int minor; - public int revision; - public boolean isSnapshot; + public final int major; + public final int minor; + public final int revision; + public final int internal; + public final boolean isSnapshot; public KylinVersion(String version) { @@ -58,12 +59,13 @@ public class KylinVersion implements Comparable { major = Integer.parseInt(splits[0]); minor = Integer.parseInt(splits[1]); - revision = splits.length < 3 ? 0 : Integer.parseInt(splits[2]); + revision = splits.length > 2 ? Integer.parseInt(splits[2]) : 0; + internal = splits.length > 3 ? Integer.parseInt(splits[3]) : 0; } @Override public String toString() { - return "" + major + "." + minor + "." + revision; + return "" + major + "." + minor + "." + revision + "." + internal + (isSnapshot ? "-SNAPSHOT" : ""); } @Override @@ -83,13 +85,17 @@ public class KylinVersion implements Comparable { if (comp != 0) return comp; + comp = this.internal - v.internal; + if (comp != 0) + return comp; + return (this.isSnapshot ? 0 : 1) - (v.isSnapshot ? 0 : 1); } /** * Require MANUAL updating kylin version per ANY upgrading. */ - private static final KylinVersion CURRENT_KYLIN_VERSION = new KylinVersion("2.1.0"); + private static final KylinVersion CURRENT_KYLIN_VERSION = new KylinVersion("2.1.0.20403"); private static final KylinVersion VERSION_200 = new KylinVersion("2.0.0"); @@ -125,6 +131,10 @@ public class KylinVersion implements Comparable { public static boolean isBefore200(String ver) { return new KylinVersion(ver).compareTo(VERSION_200) < 0; } + + public static int compare(String v1, String v2) { + return new KylinVersion(v1).compareTo(new KylinVersion(v2)); + } public static void main(String[] args) { System.out.println(getKylinClientInformation()); http://git-wip-us.apache.org/repos/asf/kylin/blob/c8bab3df/core-common/src/test/java/org/apache/kylin/common/KylinVersionTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/KylinVersionTest.java b/core-common/src/test/java/org/apache/kylin/common/KylinVersionTest.java index 96b3189..6e903c8 100644 --- a/core-common/src/test/java/org/apache/kylin/common/KylinVersionTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/KylinVersionTest.java @@ -37,4 +37,21 @@ public class KylinVersionTest { Assert.assertEquals(1, ver1.minor); Assert.assertEquals(0, ver1.revision); } -} + + @Test + public void testToString() { + KylinVersion ver1 = new KylinVersion("2.1.7.321"); + Assert.assertEquals(2, ver1.major); + Assert.assertEquals(1, ver1.minor); + Assert.assertEquals(7, ver1.revision); + Assert.assertEquals(321, ver1.internal); + Assert.assertEquals("2.1.7.321", ver1.toString()); + } + + @Test + public void testCompare() { + Assert.assertEquals(true, KylinVersion.isBefore200("1.9.9")); + Assert.assertEquals(false, KylinVersion.isBefore200("2.0.0")); + Assert.assertEquals(true, new KylinVersion("2.1.0").compareTo(new KylinVersion("2.1.0.123")) < 0); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/c8bab3df/core-cube/src/main/java/org/apache/kylin/cube/cuboid/AggregationGroupScheduler.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/AggregationGroupScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/AggregationGroupScheduler.java deleted file mode 100644 index cdcbcfc..0000000 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/AggregationGroupScheduler.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.cube.cuboid; - -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import javax.annotation.Nullable; - -import org.apache.kylin.cube.model.AggregationGroup; -import org.apache.kylin.cube.model.CubeDesc; - -import com.google.common.base.Predicate; -import com.google.common.collect.Iterators; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; - -public class AggregationGroupScheduler { - /** - * Get all valid cuboids for agg group, ignoring padding - * @param agg agg group - * @return cuboidId list - */ - public static List<Long> getCuboidsForAgg(final CubeDesc cubeDesc, AggregationGroup agg) { - Set<Long> cuboidHolder = new HashSet<>(); - - // build tree structure - Set<Long> children = getLowestCuboids(agg); - while (!children.isEmpty()) { - if (cuboidHolder.size() > cubeDesc.getConfig().getCubeAggrGroupMaxCombination()) { - throw new IllegalStateException("too many combination for the aggregation group"); - } - cuboidHolder.addAll(children); - children = getOnTreeParentsByLayer(children, agg); - } - - return Lists.newArrayList(Iterators.filter(cuboidHolder.iterator(), new Predicate<Long>() { - @Override - public boolean apply(@Nullable Long cuboidId) { - return !cubeDesc.isBlackedCuboid(cuboidId); - } - })); - } - - private static Set<Long> getOnTreeParentsByLayer(Collection<Long> children, final AggregationGroup agg) { - Set<Long> parents = new HashSet<>(); - for (long child : children) { - parents.addAll(getOnTreeParents(child, agg)); - } - parents = Sets.newHashSet(Iterators.filter(parents.iterator(), new Predicate<Long>() { - @Override - public boolean apply(@Nullable Long cuboidId) { - return agg.checkDimCap(cuboidId); - } - })); - return parents; - } - - private static Set<Long> getLowestCuboids(AggregationGroup agg) { - return getOnTreeParents(0L, agg); - } - - public static Set<Long> getOnTreeParents(long child, AggregationGroup agg) { - Set<Long> parentCandidate = new HashSet<>(); - - long tmpChild = child; - if (tmpChild == agg.getPartialCubeFullMask()) { - return parentCandidate; - } - - if (agg.getMandatoryColumnMask() != 0L) { - if (agg.isMandatoryOnlyValid()) { - if (fillBit(tmpChild, agg.getMandatoryColumnMask(), parentCandidate)) { - return parentCandidate; - } - } else { - tmpChild |= agg.getMandatoryColumnMask(); - } - } - - for (Long normal : agg.getNormalDims()) { - fillBit(tmpChild, normal, parentCandidate); - } - - for (Long joint : agg.getJoints()) { - fillBit(tmpChild, joint, parentCandidate); - } - - for (AggregationGroup.HierarchyMask hierarchy : agg.getHierarchyMasks()) { - for (long mask : hierarchy.allMasks) { - if (fillBit(tmpChild, mask, parentCandidate)) { - break; - } - } - } - - return parentCandidate; - } - - private static boolean fillBit(long origin, long other, Set<Long> coll) { - if ((origin & other) != other) { - coll.add(origin | other); - return true; - } - return false; - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/c8bab3df/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java index 1d8f589..e802230 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Set; import org.apache.kylin.common.util.ClassUtil; +import org.apache.kylin.cube.model.AggregationGroup; import org.apache.kylin.cube.model.CubeDesc; import com.google.common.base.Preconditions; @@ -62,6 +63,9 @@ abstract public class CuboidScheduler { /** Returns a cuboid on the tree that best matches the request cuboid. */ abstract public long findBestMatchCuboid(long requestCuboid); + + /** (AggGroupScheduler) Calculate the cuboid set defined by an aggregation group. */ + abstract public Set<Long> calculateCuboidsForAggGroup(AggregationGroup agg); // ============================================================================ http://git-wip-us.apache.org/repos/asf/kylin/blob/c8bab3df/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java index b75acd5..1c18b69 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java @@ -48,7 +48,6 @@ public class DefaultCuboidScheduler extends CuboidScheduler { private final Set<Long> allCuboidIds; private final Map<Long, List<Long>> parent2child; - public DefaultCuboidScheduler(CubeDesc cubeDesc) { super(cubeDesc); @@ -84,6 +83,46 @@ public class DefaultCuboidScheduler extends CuboidScheduler { } /** + * Get the parent cuboid really on the spanning tree. + * @param child an on-tree cuboid + * @return + */ + @Override + public long findBestMatchCuboid(long child) { + long parent = getOnTreeParent(child); + while (parent > 0) { + if (cubeDesc.getAllCuboids().contains(parent)) { + break; + } + parent = getOnTreeParent(parent); + } + + if (parent <= 0) { + throw new IllegalStateException("Can't find valid parent for Cuboid " + child); + } + return parent; + } + + private long getOnTreeParent(long child) { + Collection<Long> candidates = getOnTreeParents(child); + if (candidates == null || candidates.isEmpty()) { + return -1; + } + return Collections.min(candidates, Cuboid.cuboidSelectComparator); + } + + private Set<Long> getOnTreeParents(long child) { + List<AggregationGroup> aggrs = Lists.newArrayList(); + for (AggregationGroup agg : cubeDesc.getAggregationGroups()) { + if (agg.isOnTree(child)) { + aggrs.add(agg); + } + } + + return getOnTreeParents(child, aggrs); + } + + /** * Collect cuboid from bottom up, considering all factor including black list * Build tree steps: * 1. Build tree from bottom up considering dim capping @@ -94,7 +133,7 @@ public class DefaultCuboidScheduler extends CuboidScheduler { * and grandparent are missing, add grandparent to the tree. * @return Cuboid collection */ - private Pair<Set<Long>, Map<Long, List<Long>>> buildTreeBottomUp() { + protected Pair<Set<Long>, Map<Long, List<Long>>> buildTreeBottomUp() { int forward = cubeDesc.getParentForward(); KylinConfig config = cubeDesc.getConfig(); @@ -159,35 +198,6 @@ public class DefaultCuboidScheduler extends CuboidScheduler { } /** - * Get the parent cuboid really on the spanning tree. - * @param child an on-tree cuboid - * @return - */ - @Override - public long findBestMatchCuboid(long child) { - long parent = getOnTreeParent(child); - while (parent > 0) { - if (cubeDesc.getAllCuboids().contains(parent)) { - break; - } - parent = getOnTreeParent(parent); - } - - if (parent <= 0) { - throw new IllegalStateException("Can't find valid parent for Cuboid " + child); - } - return parent; - } - - private long getOnTreeParent(long child) { - Collection<Long> candidates = getOnTreeParents(child); - if (candidates == null || candidates.isEmpty()) { - return -1; - } - return Collections.min(candidates, Cuboid.cuboidSelectComparator); - } - - /** * Get all parent for children cuboids, considering dim cap. * @param children children cuboids * @return all parents cuboids @@ -205,7 +215,7 @@ public class DefaultCuboidScheduler extends CuboidScheduler { } for (AggregationGroup agg : cubeDesc.getAggregationGroups()) { - if (agg.isOnTree(cuboidId) && agg.checkDimCap(cuboidId)) { + if (agg.isOnTree(cuboidId) && checkDimCap(agg, cuboidId)) { return true; } } @@ -217,22 +227,6 @@ public class DefaultCuboidScheduler extends CuboidScheduler { } /** - * Get all *possible* parent for a child cuboid - * @param child Child cuboid ID - * @return all *possible* parent cuboids - */ - private Set<Long> getOnTreeParents(long child) { - List<AggregationGroup> aggrs = Lists.newArrayList(); - for (AggregationGroup agg : cubeDesc.getAggregationGroups()) { - if (agg.isOnTree(child)) { - aggrs.add(agg); - } - } - - return getOnTreeParents(child, aggrs); - } - - /** * Get lowest (not Cube building level) Cuboids for every Agg group * @return lowest level cuboids */ @@ -252,9 +246,108 @@ public class DefaultCuboidScheduler extends CuboidScheduler { parentCandidate.add(Cuboid.getBaseCuboidId(cubeDesc)); return parentCandidate; } - parentCandidate.addAll(AggregationGroupScheduler.getOnTreeParents(child, agg)); + parentCandidate.addAll(getOnTreeParents(child, agg)); + } + + return parentCandidate; + } + + /** + * Get all valid cuboids for agg group, ignoring padding + * @param agg agg group + * @return cuboidId list + */ + @Override + public Set<Long> calculateCuboidsForAggGroup(AggregationGroup agg) { + Set<Long> cuboidHolder = new HashSet<>(); + + // build tree structure + Set<Long> children = getLowestCuboids(agg); + while (!children.isEmpty()) { + if (cuboidHolder.size() > cubeDesc.getConfig().getCubeAggrGroupMaxCombination()) { + throw new IllegalStateException("too many combination for the aggregation group"); + } + cuboidHolder.addAll(children); + children = getOnTreeParentsByLayer(children, agg); + } + + return Sets.newHashSet(Iterators.filter(cuboidHolder.iterator(), new Predicate<Long>() { + @Override + public boolean apply(@Nullable Long cuboidId) { + return !cubeDesc.isBlackedCuboid(cuboidId); + } + })); + } + + private Set<Long> getOnTreeParentsByLayer(Collection<Long> children, final AggregationGroup agg) { + Set<Long> parents = new HashSet<>(); + for (long child : children) { + parents.addAll(getOnTreeParents(child, agg)); + } + parents = Sets.newHashSet(Iterators.filter(parents.iterator(), new Predicate<Long>() { + @Override + public boolean apply(@Nullable Long cuboidId) { + return checkDimCap(agg, cuboidId); + } + })); + return parents; + } + + private Set<Long> getLowestCuboids(AggregationGroup agg) { + return getOnTreeParents(0L, agg); + } + + private Set<Long> getOnTreeParents(long child, AggregationGroup agg) { + Set<Long> parentCandidate = new HashSet<>(); + + long tmpChild = child; + if (tmpChild == agg.getPartialCubeFullMask()) { + return parentCandidate; + } + + if (agg.getMandatoryColumnMask() != 0L) { + if (agg.isMandatoryOnlyValid()) { + if (fillBit(tmpChild, agg.getMandatoryColumnMask(), parentCandidate)) { + return parentCandidate; + } + } else { + tmpChild |= agg.getMandatoryColumnMask(); + } + } + + for (Long normal : agg.getNormalDims()) { + fillBit(tmpChild, normal, parentCandidate); + } + + for (Long joint : agg.getJoints()) { + fillBit(tmpChild, joint, parentCandidate); + } + + for (AggregationGroup.HierarchyMask hierarchy : agg.getHierarchyMasks()) { + for (long mask : hierarchy.allMasks) { + if (fillBit(tmpChild, mask, parentCandidate)) { + break; + } + } } return parentCandidate; } + + private boolean fillBit(long origin, long other, Set<Long> coll) { + if ((origin & other) != other) { + coll.add(origin | other); + return true; + } + return false; + } + + private boolean checkDimCap(AggregationGroup agg, long cuboidID) { + int dimCap = agg.getDimCap(); + if (dimCap <= 0) + return true; + + return Long.bitCount(cuboidID) <= dimCap; + } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/c8bab3df/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java index 0533ea1..af026af 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java @@ -27,7 +27,6 @@ import java.util.List; import java.util.Set; import java.util.TreeSet; -import org.apache.kylin.cube.cuboid.AggregationGroupScheduler; import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.metadata.model.TblColRef; @@ -296,7 +295,7 @@ public class AggregationGroup implements Serializable { long combination = 1; if (this.getDimCap() > 0) { - combination = AggregationGroupScheduler.getCuboidsForAgg(cubeDesc, this).size(); + combination = cubeDesc.getCuboidScheduler().calculateCuboidsForAggGroup(this).size(); } else { Set<String> includeDims = new TreeSet<>(Arrays.asList(includes)); Set<String> mandatoryDims = new TreeSet<>(Arrays.asList(selectRule.mandatoryDims)); @@ -389,13 +388,6 @@ public class AggregationGroup implements Serializable { return true; } - public boolean checkDimCap(long cuboidID) { - if (getDimCap() <= 0) { - return true; - } - return Long.bitCount(cuboidID) <= getDimCap(); - } - public void setIncludes(String[] includes) { this.includes = includes; } http://git-wip-us.apache.org/repos/asf/kylin/blob/c8bab3df/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java index 80052af..d78da9f 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java @@ -18,10 +18,14 @@ package org.apache.kylin.cube.model; +import java.io.Serializable; + import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -public class SelectRule implements java.io.Serializable { +public class SelectRule implements Serializable { + private static final long serialVersionUID = 1L; + @JsonProperty("hierarchy_dims") public String[][] hierarchyDims; @JsonProperty("mandatory_dims")
