minor changes for successfully running
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/13653bf5 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/13653bf5 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/13653bf5 Branch: refs/heads/2.x-staging Commit: 13653bf5904a6b7fb7e6d275a2c85a08379f0513 Parents: 2832193 Author: honma <[email protected]> Authored: Sun Feb 21 17:36:27 2016 +0800 Committer: honma <[email protected]> Committed: Tue Feb 23 18:26:35 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/cube/model/CubeDesc.java | 12 ++++---- .../validation/rule/AggregationGroupRule.java | 18 ++++-------- .../kylin/cube/AggregationGroupRuleTest.java | 4 +-- .../engine/mr/common/AbstractHadoopJob.java | 30 +++++++++++--------- 4 files changed, 32 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/13653bf5/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 5a345b9..de73399 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -547,7 +547,7 @@ public class CubeDesc extends RootPersistentEntity { Set<String> jointDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); getDims(jointDimsList, jointDims, agg.getSelectRule().joint_dims); - if (!includeDims.containsAll(mandatoryDims) || !containsAll(includeDims, hierarchyDimsList) || !containsAll(includeDims, jointDimsList)) { + if (!includeDims.containsAll(mandatoryDims) || !includeDims.containsAll(hierarchyDims) || !includeDims.containsAll(jointDims)) { logger.error("Aggregation group " + index + " Include dims not containing all the used dims"); throw new IllegalStateException("Aggregation group " + index + " Include dims not containing all the used dims"); } @@ -562,7 +562,7 @@ public class CubeDesc extends RootPersistentEntity { int hierarchyDimSize = hierarchyDimsList.size(); int jointDimSize = jointDimsList.size(); - if (normalDimSize + hierarchyDimSize + jointDimSize > maxSize) { + if (mandatoryDims.size() + normalDimSize + hierarchyDimSize + jointDimSize > maxSize) { logger.error("Aggregation group " + index + " has too many dimensions"); throw new IllegalStateException("Aggregation group " + index + " has too many dimensions"); } @@ -573,10 +573,12 @@ public class CubeDesc extends RootPersistentEntity { if (CollectionUtils.containsAny(mandatoryDims, jointDims)) { logger.warn("Aggregation group " + index + " mandatory dims overlap with joint dims"); } + if (CollectionUtils.containsAny(hierarchyDims, jointDims)) { logger.error("Aggregation group " + index + " hierarchy dims overlap with joint dims"); throw new IllegalStateException("Aggregation group " + index + " hierarchy dims overlap with joint dims"); } + if (hasSingle(hierarchyDimsList)) { logger.error("Aggregation group " + index + " require at least 2 dims in a hierarchy"); throw new IllegalStateException("Aggregation group " + index + " require at least 2 dims in a hierarchy"); @@ -585,6 +587,7 @@ public class CubeDesc extends RootPersistentEntity { logger.error("Aggregation group " + index + " require at least 2 dims in a joint"); throw new IllegalStateException("Aggregation group " + index + " require at least 2 dims in a joint"); } + if (hasOverlap(hierarchyDimsList, hierarchyDims)) { logger.error("Aggregation group " + index + " a dim exist in more than one hierarchy"); throw new IllegalStateException("Aggregation group " + index + " a dim exist in more than one hierarchy"); @@ -607,10 +610,9 @@ public class CubeDesc extends RootPersistentEntity { } private void getDims(ArrayList<Set<String>> dimsList, Set<String> dims, String[][] stringSets) { - Set<String> temp = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); if (stringSets != null) { for (String[] ss : stringSets) { - temp.clear(); + Set<String> temp = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); for (String s : ss) { temp.add(s); dims.add(s); @@ -820,7 +822,7 @@ public class CubeDesc extends RootPersistentEntity { } FunctionDesc func = m.getFunction(); - func.init(factTable,lookupTables); + func.init(factTable, lookupTables); allColumns.addAll(func.getParameter().getColRefs()); if (ExtendedColumnMeasureType.FUNC_RAW.equalsIgnoreCase(m.getFunction().getExpression())) { http://git-wip-us.apache.org/repos/asf/kylin/blob/13653bf5/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java index 196d2e8..45208f0 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java @@ -45,18 +45,10 @@ public class AggregationGroupRule implements IValidatorRule<CubeDesc> { if (input == null) { return 0; } else { - int count = 0; - for (String[] x : input) { - count += count(x); - } - return count; + return input.length; } } - private int count(String[] input) { - return input == null ? 0 : input.length; - } - private void inner(CubeDesc cube, ValidateContext context) { int maxSize = getMaxAgrGroupSize(); @@ -89,16 +81,18 @@ public class AggregationGroupRule implements IValidatorRule<CubeDesc> { Set<String> hierarchyDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); if (agg.getSelectRule().hierarchy_dims != null) { for (String[] ss : agg.getSelectRule().hierarchy_dims) { - for (String s : ss) + for (String s : ss) { hierarchyDims.add(s); + } } } Set<String> jointDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); if (agg.getSelectRule().joint_dims != null) { for (String[] ss : agg.getSelectRule().joint_dims) { - for (String s : ss) + for (String s : ss) { jointDims.add(s); + } } } @@ -117,7 +111,7 @@ public class AggregationGroupRule implements IValidatorRule<CubeDesc> { int hierarchySize = count(agg.getSelectRule().hierarchy_dims); int jointSize = count(agg.getSelectRule().joint_dims); - if (normalDimSize + hierarchySize + jointSize > maxSize) { + if (mandatoryDims.size() + normalDimSize + hierarchySize + jointSize > maxSize) { context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " has too many dimensions"); continue; } http://git-wip-us.apache.org/repos/asf/kylin/blob/13653bf5/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java index 2b75d48..53952c4 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java @@ -53,7 +53,7 @@ public class AggregationGroupRuleTest { AggregationGroupRule rule = new AggregationGroupRule() { @Override protected int getMaxAgrGroupSize() { - return 4; + return 2; } }; @@ -63,7 +63,7 @@ public class AggregationGroupRuleTest { ValidateContext vContext = new ValidateContext(); rule.validate(desc, vContext); vContext.print(System.out); - assertEquals(1, vContext.getResults().length); + assertTrue(vContext.getResults().length > 0); assertEquals("Aggregation group 0 has too many dimensions", (vContext.getResults()[0].getMessage())); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/13653bf5/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java index 9893cfa..7615269 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java @@ -23,8 +23,8 @@ package org.apache.kylin.engine.mr.common; * */ -import static org.apache.hadoop.util.StringUtils.*; -import java.net.URI; +import static org.apache.hadoop.util.StringUtils.formatTime; + import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -147,7 +147,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { } } - private static final String KYLIN_HIVE_DEPENDENCY_JARS = "[^,]*hive-exec.jar|[^,]*hive-metastore.jar|[^,]*hive-hcatalog-core[0-9.-]*jar"; + private static final String KYLIN_HIVE_DEPENDENCY_JARS = "[^,]*hive-exec[0-9.-]+\\.jar|[^,]*hive-metastore[0-9.-]+\\.jar|[^,]*hive-hcatalog-core[0-9.-]+\\.jar"; String filterKylinHiveDependency(String kylinHiveDependency) { if (StringUtils.isBlank(kylinHiveDependency)) @@ -210,9 +210,9 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { // yarn classpath is comma separated kylinHiveDependency = kylinHiveDependency.replace(":", ","); - logger.info("Hive Dependencies Before Filtered: "+kylinHiveDependency); + logger.info("Hive Dependencies Before Filtered: " + kylinHiveDependency); String filteredHive = filterKylinHiveDependency(kylinHiveDependency); - logger.info("Hive Dependencies After Filtered: "+filteredHive); + logger.info("Hive Dependencies After Filtered: " + filteredHive); if (kylinDependency.length() > 0) kylinDependency.append(","); @@ -223,12 +223,12 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { String mrLibDir = KylinConfig.getInstanceFromEnv().getKylinJobMRLibDir(); if (!StringUtils.isBlank(mrLibDir)) { File dirFileMRLIB = new File(mrLibDir); - if(dirFileMRLIB.exists()){ + if (dirFileMRLIB.exists()) { if (kylinDependency.length() > 0) kylinDependency.append(","); kylinDependency.append(mrLibDir); - }else{ - logger.info("The directory '"+mrLibDir+"' for 'kylin.job.mr.lib.dir' does not exist!!!"); + } else { + logger.info("The directory '" + mrLibDir + "' for 'kylin.job.mr.lib.dir' does not exist!!!"); } } @@ -374,11 +374,15 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { public static KylinConfig loadKylinPropsAndMetadata() throws IOException { File metaDir = new File("meta"); - System.setProperty(KylinConfig.KYLIN_CONF, metaDir.getAbsolutePath()); - logger.info("The absolute path for meta dir is " + metaDir.getAbsolutePath()); - KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); - kylinConfig.setMetadataUrl(metaDir.getAbsolutePath()); - return kylinConfig; + if (!metaDir.getAbsolutePath().equals(System.getProperty(KylinConfig.KYLIN_CONF))) { + System.setProperty(KylinConfig.KYLIN_CONF, metaDir.getAbsolutePath()); + logger.info("The absolute path for meta dir is " + metaDir.getAbsolutePath()); + KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); + kylinConfig.setMetadataUrl(metaDir.getAbsolutePath()); + return kylinConfig; + } else { + return KylinConfig.getInstanceFromEnv(); + } } protected void attachKylinPropsAndMetadata(TableDesc table, Configuration conf) throws IOException {
