KYLIN-1136 Distinguish fast build mode and complete build mode bug fix
fix temp Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/d256a7f3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/d256a7f3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/d256a7f3 Branch: refs/heads/2.x-staging Commit: d256a7f30f1174aa0efa74ab58ca17a329e6c7f7 Parents: fce575b Author: honma <ho...@ebay.com> Authored: Tue Nov 10 16:34:39 2015 +0800 Committer: honma <ho...@ebay.com> Committed: Mon Nov 16 10:28:24 2015 +0800 ---------------------------------------------------------------------- .../kylin/job/BuildCubeWithEngineTest.java | 114 ++++++++++++------- .../org/apache/kylin/cube/CubeManagerTest.java | 5 + ...test_kylin_cube_with_slr_left_join_desc.json | 4 +- ...t_kylin_cube_without_slr_left_join_desc.json | 6 +- 4 files changed, 86 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d256a7f3/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java index af3dc43..d2a101d 100644 --- a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java +++ b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java @@ -62,6 +62,7 @@ public class BuildCubeWithEngineTest { private CubeManager cubeManager; private DefaultScheduler scheduler; protected ExecutableManager jobService; + private static boolean fastBuildMode = true; private static final Log logger = LogFactory.getLog(BuildCubeWithEngineTest.class); @@ -84,6 +85,15 @@ public class BuildCubeWithEngineTest { public static void beforeClass() throws Exception { logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath()); ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath()); + + String fastModeStr = System.getProperty("fastBuildMode"); + if (fastModeStr != null && fastModeStr.equalsIgnoreCase("false")) { + fastBuildMode = false; + logger.info("Will not use fast build mode"); + } else { + logger.info("Will use fast build mode"); + } + System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox"); if (System.getProperty("hdp.version") == null) { throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2"); @@ -93,6 +103,7 @@ public class BuildCubeWithEngineTest { @Before public void before() throws Exception { + HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA); DeployUtil.initCliWorkDir(); @@ -128,12 +139,12 @@ public class BuildCubeWithEngineTest { } private void testInner() throws Exception { - String[] testCase = new String[] { "testInnerJoinCube", "testInnerJoinCube2" }; + String[] testCase = new String[] { "testInnerJoinCubeWithoutSlr", "testInnerJoinCubeWithSlr" }; runTestAndAssertSucceed(testCase); } private void testLeft() throws Exception { - String[] testCase = new String[] { "testLeftJoinCube", "testLeftJoinCube2" }; + String[] testCase = new String[] { "testLeftJoinCubeWithSlr", "testLeftJoinCubeWithoutSlr" }; runTestAndAssertSucceed(testCase); } @@ -180,6 +191,9 @@ public class BuildCubeWithEngineTest { final Method method = BuildCubeWithEngineTest.class.getDeclaredMethod(methodName); method.setAccessible(true); return (List<String>) method.invoke(BuildCubeWithEngineTest.this); + } catch (Exception e) { + logger.error(e.getMessage()); + throw e; } finally { countDownLatch.countDown(); } @@ -188,63 +202,71 @@ public class BuildCubeWithEngineTest { @SuppressWarnings("unused") // called by reflection - private List<String> testInnerJoinCube2() throws Exception { + private List<String> testInnerJoinCubeWithSlr() throws Exception { clearSegment("test_kylin_cube_with_slr_empty"); SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); f.setTimeZone(TimeZone.getTimeZone("GMT")); long date1 = 0; - long date2 = f.parse("2013-01-01").getTime(); + long date2 = f.parse("2015-01-01").getTime(); long date3 = f.parse("2022-01-01").getTime(); List<String> result = Lists.newArrayList(); - result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date2)); - result.add(buildSegment("test_kylin_cube_with_slr_empty", date2, date3)); + + if (fastBuildMode) { + result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date3)); + } else { + result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date2)); + result.add(buildSegment("test_kylin_cube_with_slr_empty", date2, date3));//empty segment + } return result; } @SuppressWarnings("unused") // called by reflection - private List<String> testInnerJoinCube() throws Exception { - clearSegment("test_kylin_cube_without_slr_empty"); + private List<String> testInnerJoinCubeWithoutSlr() throws Exception { + clearSegment("test_kylin_cube_without_slr_empty"); SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); f.setTimeZone(TimeZone.getTimeZone("GMT")); - - // this cube's start date is 0, end date is 20501112000000 long date1 = 0; - long date2 = f.parse("2050-01-11").getTime(); - - // this cube doesn't support incremental build, always do full build - + long date2 = f.parse("2013-01-01").getTime(); + long date3 = f.parse("2013-07-01").getTime(); + long date4 = f.parse("2022-01-01").getTime(); List<String> result = Lists.newArrayList(); - result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date2)); + + if (fastBuildMode) { + result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date4)); + } else { + result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date2)); + result.add(buildSegment("test_kylin_cube_without_slr_empty", date2, date3)); + result.add(buildSegment("test_kylin_cube_without_slr_empty", date3, date4)); + result.add(mergeSegment("test_kylin_cube_without_slr_empty", date1, date3));//don't merge all segments + } return result; + } - + @SuppressWarnings("unused") // called by reflection - private List<String> testLeftJoinCube2() throws Exception { + private List<String> testLeftJoinCubeWithoutSlr() throws Exception { SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); f.setTimeZone(TimeZone.getTimeZone("GMT")); List<String> result = Lists.newArrayList(); final String cubeName = "test_kylin_cube_without_slr_left_join_empty"; - // this cube's start date is 0, end date is 20120601000000 - long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart(); - - long dateEnd = f.parse("2012-06-01").getTime(); - clearSegment(cubeName); - result.add(buildSegment(cubeName, dateStart, dateEnd)); - // then submit an append job, start date is 20120601000000, end - // date is 20220101000000 - dateStart = dateEnd; - dateEnd = f.parse("2022-01-01").getTime(); - result.add(buildSegment(cubeName, dateStart, dateEnd)); - - // build an empty segment which doesn't have data - dateStart = dateEnd; - dateEnd = f.parse("2023-01-01").getTime(); - result.add(buildSegment(cubeName, dateStart, dateEnd)); + long date1 = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart(); + long date2 = f.parse("2012-06-01").getTime(); + long date3 = f.parse("2022-01-01").getTime(); + long date4 = f.parse("2023-01-01").getTime(); + + if (fastBuildMode) { + result.add(buildSegment("test_kylin_cube_without_slr_left_join_empty", date1, date4)); + } else { + result.add(buildSegment("test_kylin_cube_without_slr_left_join_empty", date1, date2)); + result.add(buildSegment("test_kylin_cube_without_slr_left_join_empty", date2, date3)); + result.add(buildSegment("test_kylin_cube_without_slr_left_join_empty", date3, date4));//empty segment + result.add(mergeSegment("test_kylin_cube_without_slr_left_join_empty", date1, date3));//don't merge all segments + } return result; @@ -252,18 +274,26 @@ public class BuildCubeWithEngineTest { @SuppressWarnings("unused") // called by reflection - private List<String> testLeftJoinCube() throws Exception { + private List<String> testLeftJoinCubeWithSlr() throws Exception { String cubeName = "test_kylin_cube_with_slr_left_join_empty"; clearSegment(cubeName); SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); f.setTimeZone(TimeZone.getTimeZone("GMT")); - long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart(); - long dateEnd = f.parse("2050-11-12").getTime(); + long date1 = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart(); + long date2 = f.parse("2013-01-01").getTime(); + long date3 = f.parse("2013-07-01").getTime(); + long date4 = f.parse("2022-01-01").getTime(); - // this cube's start date is 0, end date is 20501112000000 List<String> result = Lists.newArrayList(); - result.add(buildSegment(cubeName, dateStart, dateEnd)); + if (fastBuildMode) { + result.add(buildSegment(cubeName, date1, date4)); + } else { + result.add(buildSegment(cubeName, date1, date2)); + result.add(buildSegment(cubeName, date2, date3)); + result.add(buildSegment(cubeName, date3, date4)); + result.add(mergeSegment(cubeName, date1, date3));//don't merge all segments + } return result; } @@ -276,6 +306,14 @@ public class BuildCubeWithEngineTest { cubeManager.updateCube(cubeBuilder); } + private String mergeSegment(String cubeName, long startDate, long endDate) throws Exception { + CubeSegment segment = cubeManager.mergeSegments(cubeManager.getCube(cubeName), startDate, endDate, true); + DefaultChainedExecutable job = EngineFactory.createBatchMergeJob(segment, "TEST"); + jobService.addJob(job); + waitForJob(job.getId()); + return job.getId(); + } + private String buildSegment(String cubeName, long startDate, long endDate) throws Exception { CubeSegment segment = cubeManager.appendSegments(cubeManager.getCube(cubeName), endDate); DefaultChainedExecutable job = EngineFactory.createBatchCubingJob(segment, "TEST"); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d256a7f3/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java index 708bf32..fcfa67d 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java @@ -22,10 +22,13 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.TimeZone; +import com.google.common.collect.Lists; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.util.JsonUtil; @@ -55,6 +58,8 @@ public class CubeManagerTest extends LocalFileMetadataTestCase { @Test public void testBasics() throws Exception { + + CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_without_slr_ready"); CubeDesc desc = cube.getDescriptor(); System.out.println(JsonUtil.writeValueAsIndentString(desc)); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d256a7f3/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json index 8e22615..32536e1 100644 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json @@ -237,6 +237,6 @@ ] }, "notify_list": null, - "engine_type": 2, - "storage_type": 2 + "engine_type": 0, + "storage_type": 0 } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d256a7f3/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json index 08a132e..dfa62f7 100644 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json @@ -15,7 +15,7 @@ }, { "id": 1, - "name": "CATEGORY", + "name": "CATEGORY" "table": "DEFAULT.TEST_CATEGORY_GROUPINGS", "column": null, "derived": [ @@ -289,6 +289,6 @@ ] }, "notify_list": null, - "engine_type": 0, - "storage_type": 0 + "engine_type": 2, + "storage_type": 2 } \ No newline at end of file
