minor, refine cuboid stats output format
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/44b02019 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/44b02019 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/44b02019 Branch: refs/heads/2.x-staging Commit: 44b0201995db10bb5738819bbdf7ac6d463ba543 Parents: 13653bf Author: honma <[email protected]> Authored: Tue Feb 23 16:46:15 2016 +0800 Committer: honma <[email protected]> Committed: Tue Feb 23 18:26:35 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 2 +- .../org/apache/kylin/common/util/SumHelper.java | 47 ++++++ .../org/apache/kylin/common/util/BasicTest.java | 17 -- .../org/apache/kylin/cube/cuboid/Cuboid.java | 12 +- .../org/apache/kylin/cube/cuboid/CuboidCLI.java | 10 +- .../org/apache/kylin/cube/CubeDescTest.java | 160 +++++++++---------- .../kylin/engine/mr/common/CubeStatsReader.java | 83 +++++++--- .../mr/steps/FactDistinctColumnsReducer.java | 2 +- 8 files changed, 209 insertions(+), 124 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/44b02019/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index bd70249..5f9983a 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -377,7 +377,7 @@ public class KylinConfigBase implements Serializable { } public int getCubeAggrGroupMaxSize() { - return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max_size", "12")); + return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max.size", "12")); } public int getDictionaryMaxCardinality() { http://git-wip-us.apache.org/repos/asf/kylin/blob/44b02019/core-common/src/main/java/org/apache/kylin/common/util/SumHelper.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/SumHelper.java b/core-common/src/main/java/org/apache/kylin/common/util/SumHelper.java new file mode 100644 index 0000000..2a75d68 --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/util/SumHelper.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.common.util; + +import java.util.Collection; + +public class SumHelper { + public static Long sumLong(Collection<Long> input) { + Long sum = 0L; + for (Long x : input) { + sum += x; + } + return sum; + } + + public static Long sumInteger(Collection<Integer> input) { + Long sum = 0L; + for (Integer x : input) { + sum += x; + } + return sum; + } + + public static Double sumDouble(Collection<Double> input) { + Double sum = (double) 0; + for (Double x : input) { + sum += x; + } + return sum; + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/44b02019/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java index b481557..f1f5aa4 100644 --- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java @@ -79,23 +79,6 @@ public class BasicTest { @Test public void testxx() throws InterruptedException { - System.out.println(System.getProperty("skipTests").length()); - System.out.println(System.getProperty("skipXests")); - byte[][] data = new byte[10000000][]; - byte[] temp = new byte[100]; - for (int i = 0; i < 100; i++) { - temp[i] = (byte) i; - } - for (int i = 0; i < 10000000; i++) { - data[i] = new byte[100]; - } - - long wallClock = System.currentTimeMillis(); - - for (int i = 0; i < 10000000; i++) { - System.arraycopy(temp, 0, data[i], 0, 100); - } - System.out.println("Time Consumed: " + (System.currentTimeMillis() - wallClock)); } @Test http://git-wip-us.apache.org/repos/asf/kylin/blob/44b02019/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java index c1212e2..513513c 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java @@ -111,7 +111,7 @@ public class Cuboid implements Comparable<Cuboid> { } public static long translateToValidCuboid(CubeDesc cubeDesc, long cuboidID) { - if(cuboidID==getBaseCuboidId(cubeDesc)){ + if (cuboidID == getBaseCuboidId(cubeDesc)) { return cuboidID; } List<Long> candidates = Lists.newArrayList(); @@ -382,4 +382,14 @@ public class Cuboid implements Comparable<Cuboid> { } return cuboidToGridTableMapping; } + + public static String getDisplayName(long cuboidID, int dimensionCount) { + StringBuilder sb = new StringBuilder(); + String temp = Long.toString(cuboidID); + for (int i = 0; i < dimensionCount - temp.length(); i++) { + sb.append("0"); + } + sb.append(temp); + return sb.toString(); + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/44b02019/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java index ec36db6..49382bd 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java @@ -30,7 +30,6 @@ import org.apache.kylin.cube.model.CubeDesc; /** * @author yangli9 - * */ public class CuboidCLI { @@ -43,10 +42,9 @@ public class CuboidCLI { } } - public static int simulateCuboidGeneration(CubeDesc cube, boolean validate) { - CuboidScheduler scheduler = new CuboidScheduler(cube); - - long baseCuboid = Cuboid.getBaseCuboidId(cube); + public static int simulateCuboidGeneration(CubeDesc cubeDesc, boolean validate) { + CuboidScheduler scheduler = new CuboidScheduler(cubeDesc); + long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc); Collection<Long> cuboidSet = new TreeSet<Long>(); cuboidSet.add(baseCuboid); LinkedList<Long> cuboidQueue = new LinkedList<Long>(); @@ -65,7 +63,7 @@ public class CuboidCLI { if (validate) { //only run this for test purpose, performance is bad when # of dims is large - TreeSet<Long> enumCuboids = enumCalcCuboidCount(cube); + TreeSet<Long> enumCuboids = enumCalcCuboidCount(cubeDesc); System.out.println(Arrays.toString(enumCuboids.toArray(new Long[enumCuboids.size()]))); if (enumCuboids.equals(cuboidSet) == false) { throw new IllegalStateException("Expected cuboid set " + enumCuboids + "; but actual cuboid set " + cuboidSet); http://git-wip-us.apache.org/repos/asf/kylin/blob/44b02019/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java index 7013cff..141a22e 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java @@ -1,56 +1,56 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.cube; - +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.cube; + import com.google.common.collect.Maps; -import org.apache.kylin.common.util.JsonUtil; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.common.util.JsonUtil; +import org.apache.kylin.common.util.LocalFileMetadataTestCase; +import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.cube.model.SelectRule; import org.apache.kylin.metadata.MetadataManager; import org.junit.*; import org.junit.rules.ExpectedException; - + import java.util.Arrays; import java.util.HashMap; import java.util.Map; - -/** - * @author yangli9 - */ -public class CubeDescTest extends LocalFileMetadataTestCase { - + +/** + * @author yangli9 + */ +public class CubeDescTest extends LocalFileMetadataTestCase { + @Rule public ExpectedException thrown = ExpectedException.none(); - @Before - public void setUp() throws Exception { - this.createTestMetadata(); - } - - @After - public void after() throws Exception { - this.cleanupTestMetadata(); - } - - @Test + @Before + public void setUp() throws Exception { + this.createTestMetadata(); + } + + @After + public void after() throws Exception { + this.cleanupTestMetadata(); + } + + @Test public void testGoodInit() throws Exception { CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc"); cubeDesc.init(getTestConfig(), MetadataManager.getInstance(getTestConfig()).getAllTablesMap()); @@ -97,13 +97,13 @@ public class CubeDescTest extends LocalFileMetadataTestCase { thrown.expect(IllegalStateException.class); thrown.expectMessage("Aggregation group 0 has too many dimensions"); - CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc"); - try { - System.setProperty("kylin.cube.aggrgroup.max_size", "3"); - cubeDesc.validate(); - } finally { - System.clearProperty("kylin.cube.aggrgroup.max_size"); - } + CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc"); + try { + System.setProperty("kylin.cube.aggrgroup.max.size", "3"); + cubeDesc.validate(); + } finally { + System.clearProperty("kylin.cube.aggrgroup.max.size"); + } } @Test @@ -202,35 +202,35 @@ public class CubeDescTest extends LocalFileMetadataTestCase { } @Test - public void testSerialize() throws Exception { - CubeDesc desc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc"); - String str = JsonUtil.writeValueAsIndentString(desc); - System.out.println(str); - @SuppressWarnings("unused") - CubeDesc desc2 = JsonUtil.readValue(str, CubeDesc.class); - } - - @Test - public void testGetCubeDesc() throws Exception { - CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc"); - Assert.assertNotNull(cubeDesc); - } - - @Test - public void testSerializeMap() throws Exception { - Map<String, String> map = Maps.newHashMap(); - - map.put("key1", "value1"); - map.put("key2", "value2"); - - String mapStr = JsonUtil.writeValueAsString(map); - - System.out.println(mapStr); - - Map<?, ?> map2 = JsonUtil.readValue(mapStr, HashMap.class); - - Assert.assertEquals(map, map2); - - } - -} + public void testSerialize() throws Exception { + CubeDesc desc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc"); + String str = JsonUtil.writeValueAsIndentString(desc); + System.out.println(str); + @SuppressWarnings("unused") + CubeDesc desc2 = JsonUtil.readValue(str, CubeDesc.class); + } + + @Test + public void testGetCubeDesc() throws Exception { + CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc"); + Assert.assertNotNull(cubeDesc); + } + + @Test + public void testSerializeMap() throws Exception { + Map<String, String> map = Maps.newHashMap(); + + map.put("key1", "value1"); + map.put("key2", "value2"); + + String mapStr = JsonUtil.writeValueAsString(map); + + System.out.println(mapStr); + + Map<?, ?> map2 = JsonUtil.readValue(mapStr, HashMap.class); + + Assert.assertEquals(map, map2); + + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/44b02019/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java index 3aad6ae..e842c01 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java @@ -23,6 +23,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.PrintWriter; +import java.text.DecimalFormat; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -40,17 +41,19 @@ import org.apache.hadoop.io.SequenceFile.Reader; import org.apache.hadoop.io.SequenceFile.Reader.Option; import org.apache.hadoop.util.ReflectionUtils; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.util.ByteArray; import org.apache.kylin.common.util.Bytes; +import org.apache.kylin.common.util.SumHelper; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.cube.cuboid.Cuboid; +import org.apache.kylin.cube.cuboid.CuboidScheduler; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.engine.mr.HadoopUtil; import org.apache.kylin.engine.mr.steps.InMemCuboidJob; +import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter; import org.apache.kylin.metadata.datatype.DataType; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.SegmentStatusEnum; @@ -138,27 +141,11 @@ public class CubeStatsReader { public Map<Long, Double> getCuboidSizeMap() { return getCuboidSizeMapFromRowCount(seg, getCuboidRowCountMap()); } - + public double getMapperOverlapRatioOfFirstBuild() { return mapperOverlapRatioOfFirstBuild; } - public void print(PrintWriter out) { - Map<Long, Long> rowCountMap = getCuboidRowCountMap(); - Map<Long, Double> sizeMap = getCuboidSizeMap(); - List<Long> cuboids = new ArrayList<Long>(rowCountMap.keySet()); - Collections.sort(cuboids); - - out.println("============================================================================"); - out.println("Statistics of " + seg); - out.println(" Sampling percentage: " + samplingPercentage); - out.println(" Mapper overlap ratio: " + mapperOverlapRatioOfFirstBuild); - for (Long cuboid : cuboids) { - out.println(" Cuboid :\t" + rowCountMap.get(cuboid) + " rows, " + sizeMap.get(cuboid) + " MB"); - } - out.println("----------------------------------------------------------------------------"); - } - public static Map<Long, Long> getCuboidRowCountMapFromSampling(Map<Long, HyperLogLogPlusCounter> hllcMap, int samplingPercentage) { return Maps.transformValues(hllcMap, new Function<HyperLogLogPlusCounter, Long>() { @Nullable @@ -231,6 +218,66 @@ public class CubeStatsReader { return ret; } + private void print(PrintWriter out) { + Map<Long, Long> cuboidRows = getCuboidRowCountMap(); + Map<Long, Double> cuboidSizes = getCuboidSizeMap(); + List<Long> cuboids = new ArrayList<Long>(cuboidRows.keySet()); + Collections.sort(cuboids); + + out.println("============================================================================"); + out.println("Statistics of " + seg); + out.println(); + out.println("Total cuboids: " + cuboidRows.size()); + out.println("Total estimated rows: " + SumHelper.sumLong(cuboidRows.values())); + out.println("Total estimated size(MB): " + SumHelper.sumDouble(cuboidSizes.values())); + out.println("Sampling percentage: " + samplingPercentage); + out.println("Mapper overlap ratio: " + mapperOverlapRatioOfFirstBuild); + printCuboidInfoTreeEntry(cuboidRows, cuboidSizes, out); + out.println("----------------------------------------------------------------------------"); + } + + private void printCuboidInfoTreeEntry(Map<Long, Long> cuboidRows, Map<Long, Double> cuboidSizes, PrintWriter out) { + CubeDesc cubeDesc = seg.getCubeDesc(); + CuboidScheduler scheduler = new CuboidScheduler(cubeDesc); + long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc); + int dimensionCount = Long.bitCount(baseCuboid); + printCuboidInfoTree(0L, baseCuboid, scheduler, cuboidRows, cuboidSizes, dimensionCount, 0, out); + } + + private static void printCuboidInfoTree(long parent, long cuboidID, final CuboidScheduler scheduler, Map<Long, Long> cuboidRows, Map<Long, Double> cuboidSizes, int dimensionCount, int depth, PrintWriter out) { + printOneCuboidInfo(parent, cuboidID, cuboidRows, cuboidSizes, dimensionCount, depth, out); + + List<Long> children = scheduler.getSpanningCuboid(cuboidID); + Collections.sort(children); + + for (Long child : children) { + printCuboidInfoTree(parent, child, scheduler, cuboidRows, cuboidSizes, dimensionCount, depth + 1, out); + } + } + + private static void printOneCuboidInfo(long parent, long cuboidID, Map<Long, Long> cuboidRows, Map<Long, Double> cuboidSizes, int dimensionCount, int depth, PrintWriter out) { + StringBuffer sb = new StringBuffer(); + for (int i = 0; i < depth; i++) { + sb.append(" "); + } + String cuboidName = Cuboid.getDisplayName(cuboidID, dimensionCount); + sb.append("|---- Cuboid ").append(cuboidName); + + long rowCount = cuboidRows.get(cuboidID); + double size = cuboidSizes.get(cuboidID); + sb.append(", est row: ").append(rowCount).append(", est MB: ").append(formatDouble(size)); + + if (parent != -1) { + sb.append(", shrink: ").append(formatDouble(1.0 * cuboidRows.get(cuboidID) / cuboidRows.get(parent))).append("%"); + } + + out.println(sb.toString()); + } + + private static String formatDouble(double input) { + return new DecimalFormat("#.##").format(input); + } + public static void main(String[] args) throws IOException { KylinConfig config = KylinConfig.getInstanceFromEnv(); CubeInstance cube = CubeManager.getInstance(config).getCube(args[0]); http://git-wip-us.apache.org/repos/asf/kylin/blob/44b02019/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java index 62be210..f3e0290 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java @@ -190,7 +190,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<Text, Text, NullWri msg = "Samping percentage: \t" + samplingPercentage; writeLine(out, msg); - writeLine(out, "The following statistics are collected based sampling data."); + writeLine(out, "The following statistics are collected based on sampling data."); for (int i = 0; i < baseCuboidRowCountInMappers.size(); i++) { if (baseCuboidRowCountInMappers.get(i) > 0) { msg = "Base Cuboid in Mapper " + i + " row count: \t " + baseCuboidRowCountInMappers.get(i);
