KYLIN-1136 Distinguish fast build mode and complete build mode
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a9e33cde Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a9e33cde Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a9e33cde Branch: refs/heads/KYLIN-1126 Commit: a9e33cdea5d7fc3cb02e7a1b00456e8237557b81 Parents: 1dd1afd Author: honma <ho...@ebay.com> Authored: Tue Nov 10 16:34:39 2015 +0800 Committer: honma <ho...@ebay.com> Committed: Tue Nov 10 16:34:39 2015 +0800 ---------------------------------------------------------------------- .../kylin/job/BuildCubeWithEngineTest.java | 110 ++++++++++++------- 1 file changed, 73 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a9e33cde/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java index af3dc43..8864170 100644 --- a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java +++ b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java @@ -62,6 +62,7 @@ public class BuildCubeWithEngineTest { private CubeManager cubeManager; private DefaultScheduler scheduler; protected ExecutableManager jobService; + private static boolean fastBuildMode = true; private static final Log logger = LogFactory.getLog(BuildCubeWithEngineTest.class); @@ -84,6 +85,15 @@ public class BuildCubeWithEngineTest { public static void beforeClass() throws Exception { logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath()); ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath()); + + String fastModeStr = System.getProperty("fastBuildMode"); + if (fastModeStr != null && fastModeStr.equalsIgnoreCase("false")) { + fastBuildMode = false; + logger.info("Will not use fast build mode"); + } else { + logger.info("Will use fast build mode"); + } + System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox"); if (System.getProperty("hdp.version") == null) { throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2"); @@ -93,6 +103,7 @@ public class BuildCubeWithEngineTest { @Before public void before() throws Exception { + HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA); DeployUtil.initCliWorkDir(); @@ -128,12 +139,12 @@ public class BuildCubeWithEngineTest { } private void testInner() throws Exception { - String[] testCase = new String[] { "testInnerJoinCube", "testInnerJoinCube2" }; + String[] testCase = new String[] { "testInnerJoinCubeWithoutSlr", "testInnerJoinCubeWithSlr" }; runTestAndAssertSucceed(testCase); } private void testLeft() throws Exception { - String[] testCase = new String[] { "testLeftJoinCube", "testLeftJoinCube2" }; + String[] testCase = new String[] { "testLeftJoinCubeWithSlr", "testLeftJoinCubeWithoutSlr" }; runTestAndAssertSucceed(testCase); } @@ -188,63 +199,74 @@ public class BuildCubeWithEngineTest { @SuppressWarnings("unused") // called by reflection - private List<String> testInnerJoinCube2() throws Exception { + private List<String> testInnerJoinCubeWithSlr() throws Exception { clearSegment("test_kylin_cube_with_slr_empty"); SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); f.setTimeZone(TimeZone.getTimeZone("GMT")); long date1 = 0; long date2 = f.parse("2013-01-01").getTime(); - long date3 = f.parse("2022-01-01").getTime(); + long date3 = f.parse("2013-07-01").getTime(); + long date4 = f.parse("2022-01-01").getTime(); List<String> result = Lists.newArrayList(); - result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date2)); - result.add(buildSegment("test_kylin_cube_with_slr_empty", date2, date3)); + + if (fastBuildMode) { + result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date4)); + } else { + result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date2)); + result.add(buildSegment("test_kylin_cube_with_slr_empty", date2, date3)); + result.add(buildSegment("test_kylin_cube_with_slr_empty", date3, date4)); + result.add(mergeSegment("test_kylin_cube_with_slr_empty", date1, date3));//don't merge all segments + } return result; } @SuppressWarnings("unused") // called by reflection - private List<String> testInnerJoinCube() throws Exception { - clearSegment("test_kylin_cube_without_slr_empty"); + private List<String> testInnerJoinCubeWithoutSlr() throws Exception { + clearSegment("test_kylin_cube_without_slr_empty"); SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); f.setTimeZone(TimeZone.getTimeZone("GMT")); - - // this cube's start date is 0, end date is 20501112000000 long date1 = 0; - long date2 = f.parse("2050-01-11").getTime(); - - // this cube doesn't support incremental build, always do full build - + long date2 = f.parse("2013-01-01").getTime(); + long date3 = f.parse("2013-07-01").getTime(); + long date4 = f.parse("2022-01-01").getTime(); List<String> result = Lists.newArrayList(); - result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date2)); + + if (fastBuildMode) { + result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date4)); + } else { + result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date2)); + result.add(buildSegment("test_kylin_cube_without_slr_empty", date2, date3)); + result.add(buildSegment("test_kylin_cube_without_slr_empty", date3, date4)); + result.add(mergeSegment("test_kylin_cube_without_slr_empty", date1, date3));//don't merge all segments + } return result; + } - + @SuppressWarnings("unused") // called by reflection - private List<String> testLeftJoinCube2() throws Exception { + private List<String> testLeftJoinCubeWithoutSlr() throws Exception { SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); f.setTimeZone(TimeZone.getTimeZone("GMT")); List<String> result = Lists.newArrayList(); final String cubeName = "test_kylin_cube_without_slr_left_join_empty"; - // this cube's start date is 0, end date is 20120601000000 - long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart(); - - long dateEnd = f.parse("2012-06-01").getTime(); - clearSegment(cubeName); - result.add(buildSegment(cubeName, dateStart, dateEnd)); - - // then submit an append job, start date is 20120601000000, end - // date is 20220101000000 - dateStart = dateEnd; - dateEnd = f.parse("2022-01-01").getTime(); - result.add(buildSegment(cubeName, dateStart, dateEnd)); - // build an empty segment which doesn't have data - dateStart = dateEnd; - dateEnd = f.parse("2023-01-01").getTime(); - result.add(buildSegment(cubeName, dateStart, dateEnd)); + long date1 = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart(); + long date2 = f.parse("2012-06-01").getTime(); + long date3 = f.parse("2022-01-01").getTime(); + long date4 = f.parse("2023-01-01").getTime(); + + if (fastBuildMode) { + result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date4)); + } else { + result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date2)); + result.add(buildSegment("test_kylin_cube_without_slr_empty", date2, date3)); + result.add(buildSegment("test_kylin_cube_without_slr_empty", date3, date4)); + result.add(mergeSegment("test_kylin_cube_without_slr_empty", date1, date3));//don't merge all segments + } return result; @@ -252,18 +274,24 @@ public class BuildCubeWithEngineTest { @SuppressWarnings("unused") // called by reflection - private List<String> testLeftJoinCube() throws Exception { + private List<String> testLeftJoinCubeWithSlr() throws Exception { String cubeName = "test_kylin_cube_with_slr_left_join_empty"; clearSegment(cubeName); SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); f.setTimeZone(TimeZone.getTimeZone("GMT")); long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart(); - long dateEnd = f.parse("2050-11-12").getTime(); + long dateEnd = f.parse("2020-11-12").getTime(); + long dateFuture = f.parse("2021-11-12").getTime(); - // this cube's start date is 0, end date is 20501112000000 List<String> result = Lists.newArrayList(); - result.add(buildSegment(cubeName, dateStart, dateEnd)); + if (fastBuildMode) { + result.add(buildSegment(cubeName, dateStart, dateFuture)); + } else { + result.add(buildSegment(cubeName, dateStart, dateEnd)); + // build an empty segment which doesn't have data + result.add(buildSegment(cubeName, dateEnd, dateFuture)); + } return result; } @@ -276,6 +304,14 @@ public class BuildCubeWithEngineTest { cubeManager.updateCube(cubeBuilder); } + private String mergeSegment(String cubeName, long startDate, long endDate) throws Exception { + CubeSegment segment = cubeManager.mergeSegments(cubeManager.getCube(cubeName), startDate, endDate, true); + DefaultChainedExecutable job = EngineFactory.createBatchMergeJob(segment, "TEST"); + jobService.addJob(job); + waitForJob(job.getId()); + return job.getId(); + } + private String buildSegment(String cubeName, long startDate, long endDate) throws Exception { CubeSegment segment = cubeManager.appendSegments(cubeManager.getCube(cubeName), endDate); DefaultChainedExecutable job = EngineFactory.createBatchCubingJob(segment, "TEST");
