KYLIN-2821 Add unit test for hbase mapping checking
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/565fe9a7 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/565fe9a7 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/565fe9a7 Branch: refs/heads/ranger Commit: 565fe9a71e93d41c26c03cbba454af33235ea2dc Parents: c0e0b8f Author: auphyroc99 <454530...@qq.com> Authored: Wed Sep 6 11:28:14 2017 +0800 Committer: Roger Shi <rogershijich...@gmail.com> Committed: Wed Sep 6 12:59:48 2017 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/cube/model/CubeDesc.java | 8 +- .../org/apache/kylin/cube/CubeDescTest.java | 105 +++++++++++++++---- 2 files changed, 88 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/565fe9a7/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 92695a8..0af74f2 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -1001,7 +1001,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { extendedColumnToHosts.put(extendedColumn, new DeriveInfo(DeriveType.EXTENDED_COLUMN, null, hostCols, false)); } - private void initMeasureReferenceToColumnFamily() { + public void initMeasureReferenceToColumnFamily() { if (measures == null || measures.size() == 0) return; @@ -1026,11 +1026,11 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { measureIndex[i] = measureIndexLookup.get(colMeasureRefs[i]); checkState(measureIndex[i] >= 0, "measure index at (%s) not positive", i); + checkState(!measureSet.contains(colMeasureRefs[i]), "measure (%s) duplicates", colMeasureRefs[i]); + measureSet.add(colMeasureRefs[i]); + checkState(measureIndex[i] > lastMeasureIndex, "measure (%s) is not in order", colMeasureRefs[i]); lastMeasureIndex = measureIndex[i]; - - checkState(!measureSet.contains(colMeasureRefs[i]), "column (%s) duplicates", colMeasureRefs[i]); - measureSet.add(colMeasureRefs[i]); checkEachMeasureExist.set(measureIndex[i]); } http://git-wip-us.apache.org/repos/asf/kylin/blob/565fe9a7/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java index b8c19c4..269c4ee 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java @@ -43,6 +43,7 @@ import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; import org.apache.kylin.cube.model.CubeDesc.DeriveType; import org.apache.kylin.cube.model.DimensionDesc; +import org.apache.kylin.cube.model.HBaseColumnDesc; import org.apache.kylin.cube.model.SelectRule; import org.apache.kylin.cube.model.TooManyCuboidException; import org.apache.kylin.metadata.model.MeasureDesc; @@ -111,7 +112,7 @@ public class CubeDescTest extends LocalFileMetadataTestCase { CubeDesc ic = mgr.getCubeDesc("ci_inner_join_cube"); assertNotNull(lc); assertNotNull(ic); - + // assert the two CI cubes are identical apart from the left/inner difference assertEquals(lc.getDimensions().size(), ic.getDimensions().size()); for (int i = 0, n = lc.getDimensions().size(); i < n; i++) { @@ -121,11 +122,11 @@ public class CubeDescTest extends LocalFileMetadataTestCase { assertEquals(ld.getColumn(), id.getColumn()); assertArrayEquals(ld.getDerived(), id.getDerived()); } - + // To enable spark in IT, the inner cube removed the percentile measure, so ignore that particular measure List<MeasureDesc> lcMeasures = dropPercentile(lc.getMeasures()); List<MeasureDesc> icMeasures = ic.getMeasures(); - + assertEquals(lcMeasures.size(), icMeasures.size()); for (int i = 0, n = lcMeasures.size(); i < n; i++) { MeasureDesc lm = lcMeasures.get(i); @@ -134,7 +135,7 @@ public class CubeDescTest extends LocalFileMetadataTestCase { assertEquals(lm.getFunction().getFullExpression(), im.getFunction().getFullExpression()); assertEquals(lm.getFunction().getReturnType(), im.getFunction().getReturnType()); } - + assertEquals(lc.getAggregationGroups().size(), ic.getAggregationGroups().size()); for (int i = 0, n = lc.getAggregationGroups().size(); i < n; i++) { AggregationGroup lag = lc.getAggregationGroups().get(i); @@ -144,15 +145,15 @@ public class CubeDescTest extends LocalFileMetadataTestCase { assertArrayEquals(lag.getSelectRule().hierarchyDims, iag.getSelectRule().hierarchyDims); assertArrayEquals(lag.getSelectRule().jointDims, iag.getSelectRule().jointDims); } - + assertEquals(lc.listAllColumnDescs().size(), ic.listAllColumnDescs().size()); assertEquals(lc.listAllColumns().size(), ic.listAllColumns().size()); - + // test KYLIN-2440 assertTrue(lc.listAllColumns().contains(lc.getModel().findColumn("SELLER_ACCOUNT.ACCOUNT_ID"))); assertTrue(ic.listAllColumns().contains(ic.getModel().findColumn("SELLER_ACCOUNT.ACCOUNT_ID"))); } - + private List<MeasureDesc> dropPercentile(List<MeasureDesc> measures) { ArrayList<MeasureDesc> result = new ArrayList<>(); for (MeasureDesc m : measures) { @@ -197,7 +198,8 @@ public class CubeDescTest extends LocalFileMetadataTestCase { thrown.expect(IllegalStateException.class); thrown.expectMessage("Aggregation group 0 'includes' dimensions not include all the dimensions:"); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); - String[] temp = Arrays.asList(cubeDesc.getAggregationGroups().get(0).getIncludes()).subList(0, 3).toArray(new String[3]); + String[] temp = Arrays.asList(cubeDesc.getAggregationGroups().get(0).getIncludes()).subList(0, 3) + .toArray(new String[3]); cubeDesc.getAggregationGroups().get(0).setIncludes(temp); cubeDesc.init(getTestConfig()); @@ -206,7 +208,8 @@ public class CubeDescTest extends LocalFileMetadataTestCase { @Test public void testBadInit4() throws Exception { thrown.expect(TooManyCuboidException.class); - thrown.expectMessage("Aggregation group 0 of Cube Desc test_kylin_cube_with_slr_desc has too many combinations: 31. Use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max-combination' to a bigger value."); + thrown.expectMessage( + "Aggregation group 0 of Cube Desc test_kylin_cube_with_slr_desc has too many combinations: 31. Use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max-combination' to a bigger value."); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); try { @@ -221,7 +224,8 @@ public class CubeDescTest extends LocalFileMetadataTestCase { @Test public void testBadInit5() throws Exception { CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); - cubeDesc.getAggregationGroups().get(0).getSelectRule().mandatoryDims = new String[] { SELLER_ID, META_CATEG_NAME }; + cubeDesc.getAggregationGroups().get(0).getSelectRule().mandatoryDims = new String[] { SELLER_ID, + META_CATEG_NAME }; cubeDesc.init(getTestConfig()); } @@ -229,7 +233,8 @@ public class CubeDescTest extends LocalFileMetadataTestCase { @Test public void testBadInit6() throws Exception { CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); - cubeDesc.getAggregationGroups().get(0).getSelectRule().mandatoryDims = new String[] { SELLER_ID, LSTG_FORMAT_NAME }; + cubeDesc.getAggregationGroups().get(0).getSelectRule().mandatoryDims = new String[] { SELLER_ID, + LSTG_FORMAT_NAME }; cubeDesc.init(getTestConfig()); } @@ -240,7 +245,8 @@ public class CubeDescTest extends LocalFileMetadataTestCase { thrown.expectMessage("Aggregation group 0 require at least 2 dimensions in a joint"); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); - cubeDesc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] { new String[] { LSTG_FORMAT_NAME } }; + cubeDesc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] { + new String[] { LSTG_FORMAT_NAME } }; cubeDesc.init(getTestConfig()); } @@ -249,10 +255,12 @@ public class CubeDescTest extends LocalFileMetadataTestCase { public void testBadInit8() throws Exception { String[] strs = new String[] { CATEG_LVL2_NAME, META_CATEG_NAME }; thrown.expect(IllegalStateException.class); - thrown.expectMessage("Aggregation group 0 hierarchy dimensions overlap with joint dimensions: " + sortStrs(strs)); + thrown.expectMessage( + "Aggregation group 0 hierarchy dimensions overlap with joint dimensions: " + sortStrs(strs)); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); - cubeDesc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] { new String[] { META_CATEG_NAME, CATEG_LVL2_NAME } }; + cubeDesc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] { + new String[] { META_CATEG_NAME, CATEG_LVL2_NAME } }; cubeDesc.init(getTestConfig()); } @@ -261,10 +269,14 @@ public class CubeDescTest extends LocalFileMetadataTestCase { public void testBadInit9() throws Exception { String[] strs = new String[] { LSTG_FORMAT_NAME, META_CATEG_NAME }; thrown.expect(IllegalStateException.class); - thrown.expectMessage("Aggregation group 0 hierarchy dimensions overlap with joint dimensions: " + sortStrs(strs)); + thrown.expectMessage( + "Aggregation group 0 hierarchy dimensions overlap with joint dimensions: " + sortStrs(strs)); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); - cubeDesc.getAggregationGroups().get(0).getSelectRule().hierarchyDims = new String[][] { new String[] { META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME }, new String[] { LSTG_FORMAT_NAME, LSTG_SITE_ID } }; - cubeDesc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] { new String[] { META_CATEG_NAME, LSTG_FORMAT_NAME } }; + cubeDesc.getAggregationGroups().get(0).getSelectRule().hierarchyDims = new String[][] { + new String[] { META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME }, + new String[] { LSTG_FORMAT_NAME, LSTG_SITE_ID } }; + cubeDesc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] { + new String[] { META_CATEG_NAME, LSTG_FORMAT_NAME } }; cubeDesc.init(getTestConfig()); } @@ -276,7 +288,9 @@ public class CubeDescTest extends LocalFileMetadataTestCase { thrown.expectMessage("Aggregation group 0 a dimension exist in more than one joint: " + sortStrs(strs)); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); - cubeDesc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] { new String[] { LSTG_FORMAT_NAME, LSTG_SITE_ID, SLR_SEGMENT_CD }, new String[] { LSTG_FORMAT_NAME, LSTG_SITE_ID, LEAF_CATEG_ID } }; + cubeDesc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] { + new String[] { LSTG_FORMAT_NAME, LSTG_SITE_ID, SLR_SEGMENT_CD }, + new String[] { LSTG_FORMAT_NAME, LSTG_SITE_ID, LEAF_CATEG_ID } }; cubeDesc.init(getTestConfig()); } @@ -287,7 +301,8 @@ public class CubeDescTest extends LocalFileMetadataTestCase { thrown.expectMessage("Aggregation group 0 require at least 2 dimensions in a hierarchy."); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); - cubeDesc.getAggregationGroups().get(0).getSelectRule().hierarchyDims = new String[][] { new String[] { META_CATEG_NAME } }; + cubeDesc.getAggregationGroups().get(0).getSelectRule().hierarchyDims = new String[][] { + new String[] { META_CATEG_NAME } }; cubeDesc.init(getTestConfig()); } @@ -299,12 +314,59 @@ public class CubeDescTest extends LocalFileMetadataTestCase { thrown.expectMessage("Aggregation group 0 a dimension exist in more than one hierarchy: " + sortStrs(strs)); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); - cubeDesc.getAggregationGroups().get(0).getSelectRule().hierarchyDims = new String[][] { new String[] { META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME }, new String[] { META_CATEG_NAME, CATEG_LVL2_NAME } }; + cubeDesc.getAggregationGroups().get(0).getSelectRule().hierarchyDims = new String[][] { + new String[] { META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME }, + new String[] { META_CATEG_NAME, CATEG_LVL2_NAME } }; cubeDesc.init(getTestConfig()); } @Test + public void testBadInit13() throws Exception { + thrown.expect(IllegalStateException.class); + CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); + MeasureDesc measureForTransCnt = cubeDesc.getMeasures().get(3); + Assert.assertEquals(measureForTransCnt.getName(), "TRANS_CNT"); + thrown.expectMessage("measure (" + measureForTransCnt.getName() + ") is not in order"); + HBaseColumnDesc colDesc = new HBaseColumnDesc(); + colDesc.setQualifier("M"); + colDesc.setMeasureRefs(new String[] { "GMV_SUM", "GMV_MIN", "GMV_MAX", "ITEM_COUNT_SUM", "TRANS_CNT" }); + cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0] = colDesc; + cubeDesc.initMeasureReferenceToColumnFamily(); + } + + @Test + public void testBadInit14() throws Exception { + thrown.expect(IllegalStateException.class); + CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); + MeasureDesc measureForTransCnt = cubeDesc.getMeasures().get(3); + Assert.assertEquals(measureForTransCnt.getName(), "TRANS_CNT"); + String measureInfoForTransCnt = measureForTransCnt.toString(); + thrown.expectMessage( + "measure (" + measureInfoForTransCnt + ") does not exist in column family, or measure duplicates"); + HBaseColumnDesc colDesc = new HBaseColumnDesc(); + colDesc.setQualifier("M"); + colDesc.setMeasureRefs(new String[] { "GMV_SUM", "GMV_MIN", "GMV_MAX", "ITEM_COUNT_SUM" }); + cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0] = colDesc; + cubeDesc.initMeasureReferenceToColumnFamily(); + } + + @Test + public void testBadInit15() throws Exception { + thrown.expect(IllegalStateException.class); + CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); + MeasureDesc measureForTransCnt = cubeDesc.getMeasures().get(3); + Assert.assertEquals(measureForTransCnt.getName(), "TRANS_CNT"); + thrown.expectMessage("measure (" + measureForTransCnt.getName() + ") duplicates"); + HBaseColumnDesc colDesc = new HBaseColumnDesc(); + colDesc.setQualifier("M"); + colDesc.setMeasureRefs( + new String[] { "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "TRANS_CNT", "ITEM_COUNT_SUM" }); + cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0] = colDesc; + cubeDesc.initMeasureReferenceToColumnFamily(); + } + + @Test public void testCombinationIntOverflow() throws Exception { for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA, "cube_desc").listFiles()) { if (f.getName().endsWith(".bad")) { @@ -315,7 +377,8 @@ public class CubeDescTest extends LocalFileMetadataTestCase { thrown.expect(TooManyCuboidException.class); CubeDescManager.clearCache(); - CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ut_cube_desc_combination_int_overflow"); + CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()) + .getCubeDesc("ut_cube_desc_combination_int_overflow"); cubeDesc.init(getTestConfig()); }
