Repository: kylin Updated Branches: refs/heads/2.x-staging cd5076497 -> f27f7f3e1
KYLIN-976 Some measures only aggregates in BaseCuboid (Test) Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f27f7f3e Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f27f7f3e Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f27f7f3e Branch: refs/heads/2.x-staging Commit: f27f7f3e11371e9ab91f7c82dedd50986af53681 Parents: cd50764 Author: lidongsjtu <don...@ebay.com> Authored: Mon Dec 28 10:13:44 2015 +0800 Committer: lidongsjtu <don...@ebay.com> Committed: Mon Dec 28 10:13:44 2015 +0800 ---------------------------------------------------------------------- .../kylin/engine/mr/steps/CubeReducerTest.java | 85 ++++++++++ .../hbase/cube/CubeStorageQueryTest.java | 154 +++++++++++++++++++ 2 files changed, 239 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/f27f7f3e/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java index e79bd3a..aa5e77e 100644 --- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java +++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.io.File; +import java.lang.reflect.Field; import java.math.BigDecimal; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -36,8 +37,13 @@ import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.kv.RowConstants; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.engine.mr.common.BatchConstants; +import org.apache.kylin.measure.MeasureAggregator; import org.apache.kylin.measure.MeasureCodec; +import org.apache.kylin.measure.MeasureIngester; +import org.apache.kylin.measure.MeasureType; import org.apache.kylin.metadata.datatype.LongMutable; +import org.apache.kylin.metadata.model.FunctionDesc; +import org.apache.kylin.metadata.model.MeasureDesc; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -108,6 +114,53 @@ public class CubeReducerTest extends LocalFileMetadataTestCase { assertTrue(result.contains(p3)); } + @Test + public void testReducerOnlyAggrInBaseCuboid() throws Exception { + reduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr_ready"); + reduceDriver.getConfiguration().setInt(BatchConstants.CFG_CUBE_CUBOID_LEVEL, 1); + + CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor(); + MeasureDesc measureDesc = cubeDesc.getMeasures().get(0); + FunctionDesc functionDesc = measureDesc.getFunction(); + Field field = FunctionDesc.class.getDeclaredField("measureType"); + field.setAccessible(true); + MeasureType origMeasureType = functionDesc.getMeasureType(); + field.set(functionDesc, new MockUpMeasureType(origMeasureType)); + + MeasureCodec codec = new MeasureCodec(cubeDesc.getMeasures()); + + Text key1 = new Text("72010ustech"); + List<Text> values1 = new ArrayList<Text>(); + values1.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 100)); + values1.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 200)); + values1.add(newValueText(codec, "10", "10", "10", 1, 300)); + + Text key2 = new Text("1tech"); + List<Text> values2 = new ArrayList<Text>(); + values2.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 500)); + values2.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 1000)); + + Text key3 = new Text("0"); + List<Text> values3 = new ArrayList<Text>(); + values3.add(newValueText(codec, "146.52", "146.52", "146.52", 0, 0)); + + reduceDriver.withInput(key1, values1); + reduceDriver.withInput(key2, values2); + reduceDriver.withInput(key3, values3); + + List<Pair<Text, Text>> result = reduceDriver.run(); + + Pair<Text, Text> p1 = new Pair<Text, Text>(new Text("72010ustech"), newValueText(codec, "0", "10", "20.34", 3, 600)); + Pair<Text, Text> p2 = new Pair<Text, Text>(new Text("1tech"), newValueText(codec, "0", "15.09", "20.34", 2, 1500)); + Pair<Text, Text> p3 = new Pair<Text, Text>(new Text("0"), newValueText(codec, "0", "146.52", "146.52", 0, 0)); + + assertEquals(3, result.size()); + + assertTrue(result.contains(p1)); + assertTrue(result.contains(p2)); + assertTrue(result.contains(p3)); + } + private Text newValueText(MeasureCodec codec, String sum, String min, String max, int count, int item_count) { Object[] values = new Object[] { new BigDecimal(sum), new BigDecimal(min), new BigDecimal(max), new LongMutable(count), new LongMutable(item_count) }; @@ -119,4 +172,36 @@ public class CubeReducerTest extends LocalFileMetadataTestCase { return t; } + class MockUpMeasureType extends MeasureType { + MeasureType origMeasureType; + + public MockUpMeasureType(MeasureType origMeasureType) { + this.origMeasureType = origMeasureType; + } + + @Override + public boolean onlyAggrInBaseCuboid() { + return true; + } + + @Override + public MeasureIngester newIngester() { + return origMeasureType.newIngester(); + } + + @Override + public MeasureAggregator newAggregator() { + return origMeasureType.newAggregator(); + } + + @Override + public boolean needRewrite() { + return origMeasureType.needRewrite(); + } + + @Override + public Class<?> getRewriteCalciteAggrFunctionClass() { + return origMeasureType.getRewriteCalciteAggrFunctionClass(); + } + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/f27f7f3e/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/CubeStorageQueryTest.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/CubeStorageQueryTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/CubeStorageQueryTest.java new file mode 100644 index 0000000..8c90abd --- /dev/null +++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/CubeStorageQueryTest.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.storage.hbase.cube; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +import org.apache.kylin.common.util.LocalFileMetadataTestCase; +import org.apache.kylin.cube.CubeInstance; +import org.apache.kylin.cube.CubeManager; +import org.apache.kylin.cube.cuboid.Cuboid; +import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.measure.MeasureAggregator; +import org.apache.kylin.measure.MeasureIngester; +import org.apache.kylin.measure.MeasureType; +import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.model.FunctionDesc; +import org.apache.kylin.metadata.model.MeasureDesc; +import org.apache.kylin.metadata.model.TblColRef; +import org.apache.kylin.storage.ICachableStorageQuery; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + +/** + * Created by dongli on 12/28/15. + */ +public class CubeStorageQueryTest extends LocalFileMetadataTestCase { + + private CubeInstance cube; + + public CubeManager getCubeManager() { + return CubeManager.getInstance(getTestConfig()); + } + + private CubeInstance getTestKylinCubeWithSeller() { + return getCubeManager().getCube("test_kylin_cube_with_slr_empty"); + } + + @Before + public void setUp() throws Exception { + this.createTestMetadata(); + MetadataManager.clearCache(); + + cube = getTestKylinCubeWithSeller(); + } + + @After + public void after() throws Exception { + this.cleanupTestMetadata(); + } + + private void validateIdentifyCuboidOnStorageQnery(CubeDesc cubeDesc, ICachableStorageQuery query) { + long baseCuboidId = cubeDesc.getRowkey().getFullMask(); + + try { + Method method = query.getClass().getDeclaredMethod("identifyCuboid", Set.class, Collection.class); + method.setAccessible(true); + + Set<TblColRef> dimensions = Sets.newHashSet(); + List<FunctionDesc> metrics = Lists.newArrayList(); + + Object ret = method.invoke(query, dimensions, metrics); + + assertTrue(ret instanceof Cuboid); + assertNotEquals(baseCuboidId, ((Cuboid) ret).getId()); + + for (MeasureDesc measureDesc : cubeDesc.getMeasures()) { + Collections.addAll(metrics, measureDesc.getFunction()); + } + + FunctionDesc mockUpFuncDesc = new FunctionDesc(); + Field field = FunctionDesc.class.getDeclaredField("measureType"); + field.setAccessible(true); + field.set(mockUpFuncDesc, new MockUpMeasureType()); + metrics.add(mockUpFuncDesc); + + ret = method.invoke(query, dimensions, metrics); + assertEquals(baseCuboidId, ((Cuboid) ret).getId()); + + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Test + public void testIdentifyCuboidV1() { + CubeDesc cubeDesc = cube.getDescriptor(); + ICachableStorageQuery query = new org.apache.kylin.storage.hbase.cube.v1.CubeStorageQuery(cube); + validateIdentifyCuboidOnStorageQnery(cubeDesc, query); + } + + @Test + public void testIdentifyCuboidV2() { + CubeDesc cubeDesc = cube.getDescriptor(); + ICachableStorageQuery query = new org.apache.kylin.storage.hbase.cube.v2.CubeStorageQuery(cube); + validateIdentifyCuboidOnStorageQnery(cubeDesc, query); + } + + class MockUpMeasureType extends MeasureType<String> { + + @Override + public MeasureIngester<String> newIngester() { + return null; + } + + @Override + public MeasureAggregator<String> newAggregator() { + return null; + } + + @Override + public boolean needRewrite() { + return false; + } + + @Override + public Class<?> getRewriteCalciteAggrFunctionClass() { + return null; + } + + @Override + public boolean onlyAggrInBaseCuboid() { + return true; + } + } +}
