KYLIN-1356 move all it to a seprate module use exec instead
set hdp.version fix classnotdef fix classnot fix classnotdef add fastBuildMode KYLIN-1340 fix license fix exit fix jdbc test fix war attachclass fix cache KYLIN-1356 update dev support script Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1428bbc4 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1428bbc4 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1428bbc4 Branch: refs/heads/2.x-staging Commit: 1428bbc4308ceb79574a97ea2bcf24a518e5819a Parents: 6d0c8eb Author: honma <[email protected]> Authored: Tue Jan 26 12:38:05 2016 +0800 Committer: honma <[email protected]> Committed: Tue Jan 26 21:40:49 2016 +0800 ---------------------------------------------------------------------- assembly/pom.xml | 28 +- .../apache/kylin/RealizationRegistryTest.java | 57 +++ .../engine/spark/BuildCubeWithSparkTest.java | 151 ------ .../kylin/job/BuildCubeWithEngineTest.java | 333 ------------ .../kylin/job/BuildCubeWithStreamTest.java | 114 ----- .../apache/kylin/job/BuildIIWithEngineTest.java | 253 ---------- .../apache/kylin/job/BuildIIWithStreamTest.java | 290 ----------- .../kylin/job/DeployLocalMetaToRemoteTest.java | 5 +- .../hive/ITHiveSourceTableLoaderTest.java | 55 -- .../source/hive/ITHiveTableReaderTest.java | 49 -- .../source/hive/ITSnapshotManagerTest.java | 83 --- .../kylin/common/persistence/ResourceTool.java | 11 + .../org/apache/kylin/common/util/BasicTest.java | 2 + .../common/util/HBaseMetadataTestCase.java | 60 +++ .../org/apache/kylin/cube/CubeDescManager.java | 2 +- .../org/apache/kylin/cube/model/CubeDesc.java | 4 +- .../org/apache/kylin/job/dao/ExecutableDao.java | 4 +- .../cache/AbstractCacheFledgedQuery.java | 5 +- dev-support/test_all_against_hdp_2_2_4_2_2.sh | 5 +- kylin-it/pom.xml | 426 ++++++++++++++++ .../org/apache/kylin/jdbc/ITJDBCDriverTest.java | 281 ++++++++++ .../kylin/provision/BuildCubeWithEngine.java | 338 +++++++++++++ .../kylin/provision/BuildCubeWithSpark.java | 157 ++++++ .../kylin/provision/BuildCubeWithStream.java | 100 ++++ .../kylin/provision/BuildIIWithEngine.java | 254 ++++++++++ .../kylin/provision/BuildIIWithStream.java | 298 +++++++++++ .../java/org/apache/kylin/query/H2Database.java | 130 +++++ .../apache/kylin/query/ITCombinationTest.java | 78 +++ .../org/apache/kylin/query/ITIIQueryTest.java | 91 ++++ .../apache/kylin/query/ITKylinQueryTest.java | 254 ++++++++++ .../org/apache/kylin/query/KylinTestBase.java | 506 +++++++++++++++++++ .../hive/ITHiveSourceTableLoaderTest.java | 55 ++ .../source/hive/ITHiveTableReaderTest.java | 49 ++ .../source/hive/ITSnapshotManagerTest.java | 83 +++ .../storage/hbase/ITHBaseResourceStoreTest.java | 213 ++++++++ .../kylin/storage/hbase/ITHdfsOpsTest.java | 67 +++ .../kylin/storage/hbase/ITStorageTest.java | 163 ++++++ .../hbase/ii/ITInvertedIndexHBaseTest.java | 115 +++++ kylin-it/src/test/resources/logging.properties | 5 + .../src/test/resources/query/debug/query78.sql | 22 + .../src/test/resources/query/h2/query07.sql | 21 + .../src/test/resources/query/h2/query09.sql | 23 + .../src/test/resources/query/h2/query10.sql | 25 + .../src/test/resources/query/sql/query00.sql | 22 + .../src/test/resources/query/sql/query01.sql | 20 + .../src/test/resources/query/sql/query02.sql | 19 + .../src/test/resources/query/sql/query03.sql | 20 + .../src/test/resources/query/sql/query04.sql | 21 + .../src/test/resources/query/sql/query05.sql | 22 + .../src/test/resources/query/sql/query06.sql | 23 + .../src/test/resources/query/sql/query07.sql | 22 + .../src/test/resources/query/sql/query08.sql | 22 + .../src/test/resources/query/sql/query09.sql | 27 + .../src/test/resources/query/sql/query10.sql | 30 ++ .../src/test/resources/query/sql/query11.sql | 30 ++ .../src/test/resources/query/sql/query12.sql | 29 ++ .../src/test/resources/query/sql/query13.sql | 28 + .../src/test/resources/query/sql/query14.sql | 36 ++ .../src/test/resources/query/sql/query15.sql | 32 ++ .../src/test/resources/query/sql/query16.sql | 35 ++ .../src/test/resources/query/sql/query17.sql | 38 ++ .../src/test/resources/query/sql/query18.sql | 34 ++ .../src/test/resources/query/sql/query19.sql | 38 ++ .../src/test/resources/query/sql/query20.sql | 34 ++ .../src/test/resources/query/sql/query21.sql | 40 ++ .../src/test/resources/query/sql/query22.sql | 42 ++ .../src/test/resources/query/sql/query23.sql | 41 ++ .../src/test/resources/query/sql/query24.sql | 45 ++ .../src/test/resources/query/sql/query25.sql | 46 ++ .../src/test/resources/query/sql/query26.sql | 48 ++ .../src/test/resources/query/sql/query27.sql | 47 ++ .../src/test/resources/query/sql/query28.sql | 30 ++ .../src/test/resources/query/sql/query29.sql | 32 ++ .../src/test/resources/query/sql/query30.sql | 19 + .../src/test/resources/query/sql/query31.sql | 40 ++ .../src/test/resources/query/sql/query32.sql | 39 ++ .../src/test/resources/query/sql/query33.sql | 43 ++ .../src/test/resources/query/sql/query34.sql | 41 ++ .../src/test/resources/query/sql/query35.sql | 41 ++ .../src/test/resources/query/sql/query36.sql | 34 ++ .../src/test/resources/query/sql/query37.sql | 36 ++ .../src/test/resources/query/sql/query38.sql | 22 + .../src/test/resources/query/sql/query39.sql | 20 + .../src/test/resources/query/sql/query40.sql | 22 + .../src/test/resources/query/sql/query41.sql | 30 ++ .../src/test/resources/query/sql/query42.sql | 27 + .../src/test/resources/query/sql/query43.sql | 29 ++ .../src/test/resources/query/sql/query44.sql | 31 ++ .../src/test/resources/query/sql/query45.sql | 24 + .../src/test/resources/query/sql/query46.sql | 19 + .../src/test/resources/query/sql/query47.sql | 19 + .../src/test/resources/query/sql/query48.sql | 19 + .../src/test/resources/query/sql/query49.sql | 36 ++ .../src/test/resources/query/sql/query50.sql | 29 ++ .../src/test/resources/query/sql/query51.sql | 29 ++ .../src/test/resources/query/sql/query52.sql | 22 + .../src/test/resources/query/sql/query53.sql | 24 + .../src/test/resources/query/sql/query54.sql | 29 ++ .../src/test/resources/query/sql/query55.sql | 19 + .../src/test/resources/query/sql/query56.sql | 30 ++ .../src/test/resources/query/sql/query57.sql | 30 ++ .../src/test/resources/query/sql/query58.sql | 30 ++ .../src/test/resources/query/sql/query59.sql | 19 + .../src/test/resources/query/sql/query60.sql | 29 ++ .../src/test/resources/query/sql/query61.sql | 29 ++ .../src/test/resources/query/sql/query62.sql | 21 + .../src/test/resources/query/sql/query63.sql | 20 + .../src/test/resources/query/sql/query64.sql | 21 + .../src/test/resources/query/sql/query65.sql | 19 + .../src/test/resources/query/sql/query66.sql | 24 + .../src/test/resources/query/sql/query67.sql | 22 + .../src/test/resources/query/sql/query68.sql | 26 + .../src/test/resources/query/sql/query69.sql | 25 + .../src/test/resources/query/sql/query70.sql | 24 + .../src/test/resources/query/sql/query71.sql | 24 + .../src/test/resources/query/sql/query72.sql | 28 + .../src/test/resources/query/sql/query73.sql | 19 + .../src/test/resources/query/sql/query74.sql | 19 + .../src/test/resources/query/sql/query75.sql | 22 + .../src/test/resources/query/sql/query76.sql | 22 + .../src/test/resources/query/sql/query77.sql | 21 + .../src/test/resources/query/sql/query78.sql | 22 + .../src/test/resources/query/sql/query79.sql | 19 + .../src/test/resources/query/sql/query80.sql | 28 + .../src/test/resources/query/sql/query85.sql | 26 + .../src/test/resources/query/sql/query86.sql | 24 + .../src/test/resources/query/sql/query87.sql | 22 + .../src/test/resources/query/sql/query88.sql | 22 + .../src/test/resources/query/sql/query89.sql | 22 + .../src/test/resources/query/sql/query90.sql | 24 + .../src/test/resources/query/sql/query91.sql | 24 + .../resources/query/sql/query92.sql.disabled | 30 ++ .../resources/query/sql/query93.sql.disabled | 30 ++ .../resources/query/sql/query94.sql.disabled | 30 ++ .../resources/query/sql/query95.sql.disabled | 30 ++ .../src/test/resources/query/sql/query96.sql | 22 + .../src/test/resources/query/sql/query97.sql | 28 + .../resources/query/sql_all_types/query00.sql | 19 + .../resources/query/sql_all_types/query01.sql | 19 + .../resources/query/sql_all_types/query02.sql | 19 + .../resources/query/sql_all_types/query03.sql | 19 + .../resources/query/sql_all_types/query04.sql | 19 + .../resources/query/sql_all_types/query05.sql | 19 + .../resources/query/sql_all_types/query06.sql | 19 + .../resources/query/sql_all_types/query07.sql | 19 + .../resources/query/sql_all_types/query08.sql | 19 + .../resources/query/sql_all_types/query09.sql | 19 + .../test/resources/query/sql_cache/query01.sql | 11 + .../test/resources/query/sql_cache/query02.sql | 11 + .../test/resources/query/sql_cache/query03.sql | 11 + .../test/resources/query/sql_cache/query04.sql | 11 + .../test/resources/query/sql_cache/query05.sql | 11 + .../test/resources/query/sql_cache/query06.sql | 11 + .../test/resources/query/sql_cache/query07.sql | 11 + .../test/resources/query/sql_cache/query08.sql | 11 + .../test/resources/query/sql_cache/query09.sql | 11 + .../resources/query/sql_casewhen/query01.sql | 26 + .../resources/query/sql_casewhen/query02.sql | 22 + .../resources/query/sql_casewhen/query03.sql | 25 + .../query/sql_casewhen/query55.sql.disable | 22 + .../resources/query/sql_casewhen/query56.sql | 22 + .../resources/query/sql_derived/query01.sql | 47 ++ .../resources/query/sql_derived/query02.sql | 47 ++ .../resources/query/sql_derived/query03.sql | 21 + .../resources/query/sql_derived/query04.sql | 20 + .../resources/query/sql_derived/query05.sql | 28 + .../resources/query/sql_derived/query06.sql | 20 + .../resources/query/sql_derived/query07.sql | 22 + .../query/sql_derived/query08.sql.disabled | 19 + .../resources/query/sql_derived/query09.sql | 19 + .../resources/query/sql_derived/query10.sql | 44 ++ .../resources/query/sql_distinct/query00.sql | 24 + .../resources/query/sql_distinct/query01.sql | 25 + .../resources/query/sql_distinct/query02.sql | 26 + .../resources/query/sql_distinct/query03.sql | 26 + .../resources/query/sql_distinct/query04.sql | 27 + .../resources/query/sql_distinct/query05.sql | 25 + .../resources/query/sql_distinct/query06.sql | 26 + .../resources/query/sql_distinct/query07.sql | 24 + .../query/sql_distinct_precisely/query00.sql | 24 + .../query/sql_distinct_precisely/query01.sql | 25 + .../query/sql_distinct_precisely/query02.sql | 26 + .../query/sql_distinct_precisely/query03.sql | 33 ++ .../query/sql_distinct_precisely/query04.sql | 34 ++ .../query/sql_distinct_precisely/query05.sql | 25 + .../query/sql_distinct_precisely/query06.sql | 26 + .../query/sql_distinct_precisely/query07.sql | 24 + .../resources/query/sql_dynamic/query01.dat | 2 + .../resources/query/sql_dynamic/query01.sql | 28 + .../resources/query/sql_dynamic/query02.dat | 2 + .../resources/query/sql_dynamic/query02.sql | 30 ++ .../resources/query/sql_fast_common/query00.sql | 23 + .../test/resources/query/sql_hive/query01.sql | 20 + .../test/resources/query/sql_hive/query10.sql | 26 + .../src/test/resources/query/sql_ii/query01.sql | 19 + .../src/test/resources/query/sql_ii/query02.sql | 19 + .../src/test/resources/query/sql_ii/query03.sql | 19 + .../src/test/resources/query/sql_ii/query04.sql | 19 + .../src/test/resources/query/sql_ii/query05.sql | 19 + .../resources/query/sql_ii/query06.sql.disabled | 19 + .../resources/query/sql_ii/query07.sql.disabled | 19 + .../src/test/resources/query/sql_ii/query08.sql | 19 + .../src/test/resources/query/sql_ii/query09.sql | 19 + .../query/sql_invalid/query29_invalid_SQL.sql | 55 ++ .../query/sql_invalid/query30_invalid_SQL.sql | 54 ++ .../query/sql_invalid/query31_invalid_SQL.sql | 55 ++ .../query/sql_invalid/query32_invalid_SQL.sql | 56 ++ .../query/sql_invalid/query33_invalid_SQL.sql | 56 ++ .../query/sql_invalid/query34_invalid_SQL.sql | 24 + ...ery_count_distinct_on_dimension.sql.disabled | 20 + .../test/resources/query/sql_lookup/query01.sql | 24 + .../test/resources/query/sql_lookup/query02.sql | 19 + .../test/resources/query/sql_lookup/query03.sql | 19 + .../test/resources/query/sql_lookup/query04.sql | 19 + .../query/sql_optimize/enable-limit01.sql | 19 + .../resources/query/sql_orderby/query01.sql | 32 ++ .../resources/query/sql_orderby/query02.sql | 25 + .../resources/query/sql_orderby/query03.sql | 24 + .../resources/query/sql_streaming/query01.sql | 1 + .../resources/query/sql_streaming/query02.sql | 1 + .../resources/query/sql_streaming/query03.sql | 1 + .../resources/query/sql_streaming/query04.sql | 1 + .../resources/query/sql_streaming/query05.sql | 1 + .../resources/query/sql_streaming/query06.sql | 1 + .../resources/query/sql_streaming/query07.sql | 1 + .../resources/query/sql_streaming/query08.sql | 1 + .../resources/query/sql_streaming/query09.sql | 1 + .../resources/query/sql_streaming/query10.sql | 1 + .../resources/query/sql_subquery/query00.sql | 27 + .../resources/query/sql_subquery/query01.sql | 31 ++ .../query/sql_subquery/query02.sql.disable | 25 + .../resources/query/sql_subquery/query03.sql | 36 ++ .../resources/query/sql_subquery/query04.sql | 36 ++ .../resources/query/sql_subquery/query05.sql | 37 ++ .../resources/query/sql_subquery/query06.sql | 24 + .../resources/query/sql_subquery/query07.sql | 24 + .../resources/query/sql_subquery/query08.sql | 42 ++ .../resources/query/sql_tableau/query00.sql | 23 + .../resources/query/sql_tableau/query01.sql | 33 ++ .../resources/query/sql_tableau/query02.sql | 30 ++ .../resources/query/sql_tableau/query03.sql | 35 ++ .../resources/query/sql_tableau/query04.sql | 39 ++ .../resources/query/sql_tableau/query05.sql | 40 ++ .../resources/query/sql_tableau/query06.sql | 40 ++ .../resources/query/sql_tableau/query07.sql | 26 + .../query/sql_tableau/query08.sql.disabled | 24 + .../query/sql_tableau/query09.sql.disabled | 57 +++ .../resources/query/sql_tableau/query10.sql | 23 + .../resources/query/sql_tableau/query11.sql | 24 + .../resources/query/sql_tableau/query12.sql | 23 + .../resources/query/sql_tableau/query13.sql | 22 + .../resources/query/sql_tableau/query14.sql | 19 + .../resources/query/sql_tableau/query15.sql | 22 + .../resources/query/sql_tableau/query16.sql | 22 + .../resources/query/sql_tableau/query17.sql | 19 + .../resources/query/sql_tableau/query18.sql | 21 + .../resources/query/sql_tableau/query19.sql | 26 + .../resources/query/sql_tableau/query20.sql | 26 + .../resources/query/sql_tableau/query21.sql | 27 + .../query/sql_tableau/query22.sql.disabled | 28 + .../resources/query/sql_tableau/query23.sql | 23 + .../resources/query/sql_tableau/query24.sql | 24 + .../resources/query/sql_tableau/query25.sql | 28 + .../resources/query/sql_tableau/query27.sql | 24 + .../resources/query/sql_tableau/query28.sql | 23 + .../resources/query/sql_tableau/query67.sql | 23 + .../test/resources/query/sql_topn/query81.sql | 28 + .../test/resources/query/sql_topn/query82.sql | 27 + .../test/resources/query/sql_topn/query83.sql | 29 ++ .../resources/query/sql_verifyCount/query04.sql | 19 + .../query/sql_verifyCount/query04.sql.expected | 1 + .../resources/query/sql_verifyCount/query05.sql | 19 + .../query/sql_verifyCount/query05.sql.expected | 1 + .../resources/query/sql_verifyCount/query06.sql | 19 + .../query/sql_verifyCount/query06.sql.expected | 1 + .../resources/query/sql_verifyCount/query07.sql | 19 + .../query/sql_verifyCount/query07.sql.expected | 1 + .../resources/query/sql_verifyCount/query08.sql | 19 + .../query/sql_verifyCount/query08.sql.expected | 1 + .../resources/query/tableau_probing/query01.sql | 19 + .../resources/query/tableau_probing/query02.sql | 19 + .../resources/query/tableau_probing/query03.sql | 19 + .../resources/query/tableau_probing/query04.sql | 19 + .../resources/query/tableau_probing/query05.sql | 19 + .../resources/query/tableau_probing/query06.sql | 19 + .../resources/query/tableau_probing/query07.sql | 19 + .../src/test/resources/query/temp/.gitignore | 1 + pom.xml | 74 +-- query/pom.xml | 68 +-- .../org/apache/kylin/query/test/H2Database.java | 130 ----- .../kylin/query/test/ITCombinationTest.java | 78 --- .../apache/kylin/query/test/ITIIQueryTest.java | 92 ---- .../kylin/query/test/ITKylinQueryTest.java | 254 ---------- .../apache/kylin/query/test/KylinTestBase.java | 506 ------------------- .../query/test/RealizationRegistryTest.java | 57 --- query/src/test/resources/logging.properties | 5 - .../src/test/resources/query/debug/query78.sql | 22 - query/src/test/resources/query/h2/query07.sql | 21 - query/src/test/resources/query/h2/query09.sql | 23 - query/src/test/resources/query/h2/query10.sql | 25 - query/src/test/resources/query/sql/query00.sql | 22 - query/src/test/resources/query/sql/query01.sql | 20 - query/src/test/resources/query/sql/query02.sql | 19 - query/src/test/resources/query/sql/query03.sql | 20 - query/src/test/resources/query/sql/query04.sql | 21 - query/src/test/resources/query/sql/query05.sql | 22 - query/src/test/resources/query/sql/query06.sql | 23 - query/src/test/resources/query/sql/query07.sql | 22 - query/src/test/resources/query/sql/query08.sql | 22 - query/src/test/resources/query/sql/query09.sql | 27 - query/src/test/resources/query/sql/query10.sql | 30 -- query/src/test/resources/query/sql/query11.sql | 30 -- query/src/test/resources/query/sql/query12.sql | 29 -- query/src/test/resources/query/sql/query13.sql | 28 - query/src/test/resources/query/sql/query14.sql | 36 -- query/src/test/resources/query/sql/query15.sql | 32 -- query/src/test/resources/query/sql/query16.sql | 35 -- query/src/test/resources/query/sql/query17.sql | 38 -- query/src/test/resources/query/sql/query18.sql | 34 -- query/src/test/resources/query/sql/query19.sql | 38 -- query/src/test/resources/query/sql/query20.sql | 34 -- query/src/test/resources/query/sql/query21.sql | 40 -- query/src/test/resources/query/sql/query22.sql | 42 -- query/src/test/resources/query/sql/query23.sql | 41 -- query/src/test/resources/query/sql/query24.sql | 45 -- query/src/test/resources/query/sql/query25.sql | 46 -- query/src/test/resources/query/sql/query26.sql | 48 -- query/src/test/resources/query/sql/query27.sql | 47 -- query/src/test/resources/query/sql/query28.sql | 30 -- query/src/test/resources/query/sql/query29.sql | 32 -- query/src/test/resources/query/sql/query30.sql | 19 - query/src/test/resources/query/sql/query31.sql | 40 -- query/src/test/resources/query/sql/query32.sql | 39 -- query/src/test/resources/query/sql/query33.sql | 43 -- query/src/test/resources/query/sql/query34.sql | 41 -- query/src/test/resources/query/sql/query35.sql | 41 -- query/src/test/resources/query/sql/query36.sql | 34 -- query/src/test/resources/query/sql/query37.sql | 36 -- query/src/test/resources/query/sql/query38.sql | 22 - query/src/test/resources/query/sql/query39.sql | 20 - query/src/test/resources/query/sql/query40.sql | 22 - query/src/test/resources/query/sql/query41.sql | 30 -- query/src/test/resources/query/sql/query42.sql | 27 - query/src/test/resources/query/sql/query43.sql | 29 -- query/src/test/resources/query/sql/query44.sql | 31 -- query/src/test/resources/query/sql/query45.sql | 24 - query/src/test/resources/query/sql/query46.sql | 19 - query/src/test/resources/query/sql/query47.sql | 19 - query/src/test/resources/query/sql/query48.sql | 19 - query/src/test/resources/query/sql/query49.sql | 36 -- query/src/test/resources/query/sql/query50.sql | 29 -- query/src/test/resources/query/sql/query51.sql | 29 -- query/src/test/resources/query/sql/query52.sql | 22 - query/src/test/resources/query/sql/query53.sql | 24 - query/src/test/resources/query/sql/query54.sql | 29 -- query/src/test/resources/query/sql/query55.sql | 19 - query/src/test/resources/query/sql/query56.sql | 30 -- query/src/test/resources/query/sql/query57.sql | 30 -- query/src/test/resources/query/sql/query58.sql | 30 -- query/src/test/resources/query/sql/query59.sql | 19 - query/src/test/resources/query/sql/query60.sql | 29 -- query/src/test/resources/query/sql/query61.sql | 29 -- query/src/test/resources/query/sql/query62.sql | 21 - query/src/test/resources/query/sql/query63.sql | 20 - query/src/test/resources/query/sql/query64.sql | 21 - query/src/test/resources/query/sql/query65.sql | 19 - query/src/test/resources/query/sql/query66.sql | 24 - query/src/test/resources/query/sql/query67.sql | 22 - query/src/test/resources/query/sql/query68.sql | 26 - query/src/test/resources/query/sql/query69.sql | 25 - query/src/test/resources/query/sql/query70.sql | 24 - query/src/test/resources/query/sql/query71.sql | 24 - query/src/test/resources/query/sql/query72.sql | 28 - query/src/test/resources/query/sql/query73.sql | 19 - query/src/test/resources/query/sql/query74.sql | 19 - query/src/test/resources/query/sql/query75.sql | 22 - query/src/test/resources/query/sql/query76.sql | 22 - query/src/test/resources/query/sql/query77.sql | 21 - query/src/test/resources/query/sql/query78.sql | 22 - query/src/test/resources/query/sql/query79.sql | 19 - query/src/test/resources/query/sql/query80.sql | 28 - query/src/test/resources/query/sql/query85.sql | 26 - query/src/test/resources/query/sql/query86.sql | 24 - query/src/test/resources/query/sql/query87.sql | 22 - query/src/test/resources/query/sql/query88.sql | 22 - query/src/test/resources/query/sql/query89.sql | 22 - query/src/test/resources/query/sql/query90.sql | 24 - query/src/test/resources/query/sql/query91.sql | 24 - .../resources/query/sql/query92.sql.disabled | 30 -- .../resources/query/sql/query93.sql.disabled | 30 -- .../resources/query/sql/query94.sql.disabled | 30 -- .../resources/query/sql/query95.sql.disabled | 30 -- query/src/test/resources/query/sql/query96.sql | 22 - query/src/test/resources/query/sql/query97.sql | 28 - .../resources/query/sql_all_types/query00.sql | 19 - .../resources/query/sql_all_types/query01.sql | 19 - .../resources/query/sql_all_types/query02.sql | 19 - .../resources/query/sql_all_types/query03.sql | 19 - .../resources/query/sql_all_types/query04.sql | 19 - .../resources/query/sql_all_types/query05.sql | 19 - .../resources/query/sql_all_types/query06.sql | 19 - .../resources/query/sql_all_types/query07.sql | 19 - .../resources/query/sql_all_types/query08.sql | 19 - .../resources/query/sql_all_types/query09.sql | 19 - .../test/resources/query/sql_cache/query01.sql | 11 - .../test/resources/query/sql_cache/query02.sql | 11 - .../test/resources/query/sql_cache/query03.sql | 11 - .../test/resources/query/sql_cache/query04.sql | 11 - .../test/resources/query/sql_cache/query05.sql | 11 - .../test/resources/query/sql_cache/query06.sql | 11 - .../test/resources/query/sql_cache/query07.sql | 11 - .../test/resources/query/sql_cache/query08.sql | 11 - .../test/resources/query/sql_cache/query09.sql | 11 - .../resources/query/sql_casewhen/query01.sql | 26 - .../resources/query/sql_casewhen/query02.sql | 22 - .../resources/query/sql_casewhen/query03.sql | 25 - .../query/sql_casewhen/query55.sql.disable | 22 - .../resources/query/sql_casewhen/query56.sql | 22 - .../resources/query/sql_derived/query01.sql | 47 -- .../resources/query/sql_derived/query02.sql | 47 -- .../resources/query/sql_derived/query03.sql | 21 - .../resources/query/sql_derived/query04.sql | 20 - .../resources/query/sql_derived/query05.sql | 28 - .../resources/query/sql_derived/query06.sql | 20 - .../resources/query/sql_derived/query07.sql | 22 - .../query/sql_derived/query08.sql.disabled | 19 - .../resources/query/sql_derived/query09.sql | 19 - .../resources/query/sql_derived/query10.sql | 44 -- .../resources/query/sql_distinct/query00.sql | 24 - .../resources/query/sql_distinct/query01.sql | 25 - .../resources/query/sql_distinct/query02.sql | 26 - .../resources/query/sql_distinct/query03.sql | 26 - .../resources/query/sql_distinct/query04.sql | 27 - .../resources/query/sql_distinct/query05.sql | 25 - .../resources/query/sql_distinct/query06.sql | 26 - .../resources/query/sql_distinct/query07.sql | 24 - .../query/sql_distinct_precisely/query00.sql | 24 - .../query/sql_distinct_precisely/query01.sql | 25 - .../query/sql_distinct_precisely/query02.sql | 26 - .../query/sql_distinct_precisely/query03.sql | 33 -- .../query/sql_distinct_precisely/query04.sql | 34 -- .../query/sql_distinct_precisely/query05.sql | 25 - .../query/sql_distinct_precisely/query06.sql | 26 - .../query/sql_distinct_precisely/query07.sql | 24 - .../resources/query/sql_dynamic/query01.dat | 2 - .../resources/query/sql_dynamic/query01.sql | 28 - .../resources/query/sql_dynamic/query02.dat | 2 - .../resources/query/sql_dynamic/query02.sql | 30 -- .../resources/query/sql_fast_common/query00.sql | 23 - .../test/resources/query/sql_hive/query01.sql | 20 - .../test/resources/query/sql_hive/query10.sql | 26 - .../src/test/resources/query/sql_ii/query01.sql | 19 - .../src/test/resources/query/sql_ii/query02.sql | 19 - .../src/test/resources/query/sql_ii/query03.sql | 19 - .../src/test/resources/query/sql_ii/query04.sql | 19 - .../src/test/resources/query/sql_ii/query05.sql | 19 - .../resources/query/sql_ii/query06.sql.disabled | 19 - .../resources/query/sql_ii/query07.sql.disabled | 19 - .../src/test/resources/query/sql_ii/query08.sql | 19 - .../src/test/resources/query/sql_ii/query09.sql | 19 - .../query/sql_invalid/query29_invalid_SQL.sql | 55 -- .../query/sql_invalid/query30_invalid_SQL.sql | 54 -- .../query/sql_invalid/query31_invalid_SQL.sql | 55 -- .../query/sql_invalid/query32_invalid_SQL.sql | 56 -- .../query/sql_invalid/query33_invalid_SQL.sql | 56 -- .../query/sql_invalid/query34_invalid_SQL.sql | 24 - ...ery_count_distinct_on_dimension.sql.disabled | 20 - .../test/resources/query/sql_lookup/query01.sql | 24 - .../test/resources/query/sql_lookup/query02.sql | 19 - .../test/resources/query/sql_lookup/query03.sql | 19 - .../test/resources/query/sql_lookup/query04.sql | 19 - .../query/sql_optimize/enable-limit01.sql | 19 - .../resources/query/sql_orderby/query01.sql | 32 -- .../resources/query/sql_orderby/query02.sql | 25 - .../resources/query/sql_orderby/query03.sql | 24 - .../resources/query/sql_streaming/query01.sql | 1 - .../resources/query/sql_streaming/query02.sql | 1 - .../resources/query/sql_streaming/query03.sql | 1 - .../resources/query/sql_streaming/query04.sql | 1 - .../resources/query/sql_streaming/query05.sql | 1 - .../resources/query/sql_streaming/query06.sql | 1 - .../resources/query/sql_streaming/query07.sql | 1 - .../resources/query/sql_streaming/query08.sql | 1 - .../resources/query/sql_streaming/query09.sql | 1 - .../resources/query/sql_streaming/query10.sql | 1 - .../resources/query/sql_subquery/query00.sql | 27 - .../resources/query/sql_subquery/query01.sql | 31 -- .../query/sql_subquery/query02.sql.disable | 25 - .../resources/query/sql_subquery/query03.sql | 36 -- .../resources/query/sql_subquery/query04.sql | 36 -- .../resources/query/sql_subquery/query05.sql | 37 -- .../resources/query/sql_subquery/query06.sql | 24 - .../resources/query/sql_subquery/query07.sql | 24 - .../resources/query/sql_subquery/query08.sql | 42 -- .../resources/query/sql_tableau/query00.sql | 23 - .../resources/query/sql_tableau/query01.sql | 33 -- .../resources/query/sql_tableau/query02.sql | 30 -- .../resources/query/sql_tableau/query03.sql | 35 -- .../resources/query/sql_tableau/query04.sql | 39 -- .../resources/query/sql_tableau/query05.sql | 40 -- .../resources/query/sql_tableau/query06.sql | 40 -- .../resources/query/sql_tableau/query07.sql | 26 - .../query/sql_tableau/query08.sql.disabled | 24 - .../query/sql_tableau/query09.sql.disabled | 57 --- .../resources/query/sql_tableau/query10.sql | 23 - .../resources/query/sql_tableau/query11.sql | 24 - .../resources/query/sql_tableau/query12.sql | 23 - .../resources/query/sql_tableau/query13.sql | 22 - .../resources/query/sql_tableau/query14.sql | 19 - .../resources/query/sql_tableau/query15.sql | 22 - .../resources/query/sql_tableau/query16.sql | 22 - .../resources/query/sql_tableau/query17.sql | 19 - .../resources/query/sql_tableau/query18.sql | 21 - .../resources/query/sql_tableau/query19.sql | 26 - .../resources/query/sql_tableau/query20.sql | 26 - .../resources/query/sql_tableau/query21.sql | 27 - .../query/sql_tableau/query22.sql.disabled | 28 - .../resources/query/sql_tableau/query23.sql | 23 - .../resources/query/sql_tableau/query24.sql | 24 - .../resources/query/sql_tableau/query25.sql | 28 - .../resources/query/sql_tableau/query27.sql | 24 - .../resources/query/sql_tableau/query28.sql | 23 - .../resources/query/sql_tableau/query67.sql | 23 - .../test/resources/query/sql_topn/query81.sql | 28 - .../test/resources/query/sql_topn/query82.sql | 27 - .../test/resources/query/sql_topn/query83.sql | 29 -- .../resources/query/sql_verifyCount/query04.sql | 19 - .../query/sql_verifyCount/query04.sql.expected | 1 - .../resources/query/sql_verifyCount/query05.sql | 19 - .../query/sql_verifyCount/query05.sql.expected | 1 - .../resources/query/sql_verifyCount/query06.sql | 19 - .../query/sql_verifyCount/query06.sql.expected | 1 - .../resources/query/sql_verifyCount/query07.sql | 19 - .../query/sql_verifyCount/query07.sql.expected | 1 - .../resources/query/sql_verifyCount/query08.sql | 19 - .../query/sql_verifyCount/query08.sql.expected | 1 - .../resources/query/tableau_probing/query01.sql | 19 - .../resources/query/tableau_probing/query02.sql | 19 - .../resources/query/tableau_probing/query03.sql | 19 - .../resources/query/tableau_probing/query04.sql | 19 - .../resources/query/tableau_probing/query05.sql | 19 - .../resources/query/tableau_probing/query06.sql | 19 - .../resources/query/tableau_probing/query07.sql | 19 - query/src/test/resources/query/temp/.gitignore | 1 - server/pom.xml | 19 +- .../java/org/apache/kylin/rest/DebugTomcat.java | 5 +- .../org/apache/kylin/jdbc/ITJDBCDriverTest.java | 281 ---------- .../rest/controller/ITTableControllerTest.java | 75 --- .../storage/hbase/util/CubeMetaExtractor.java | 284 +++++++++++ .../storage/hbase/common/ITStorageTest.java | 163 ------ .../hbase/ii/ITInvertedIndexHBaseTest.java | 116 ----- .../hbase/steps/HBaseMetadataTestCase.java | 63 --- .../hbase/steps/ITHBaseResourceStoreTest.java | 211 -------- .../storage/hbase/steps/ITHdfsOpsTest.java | 66 --- .../hbase/steps/SandboxMetastoreCLI.java | 6 +- 555 files changed, 10178 insertions(+), 9617 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/1428bbc4/assembly/pom.xml ---------------------------------------------------------------------- diff --git a/assembly/pom.xml b/assembly/pom.xml index 999226a..c312f41 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -44,13 +44,13 @@ <artifactId>kylin-engine-streaming</artifactId> <version>${project.parent.version}</version> </dependency> - + <dependency> <groupId>org.apache.kylin</groupId> <artifactId>kylin-invertedindex</artifactId> <version>${project.parent.version}</version> </dependency> - + <!-- Env & Test --> <dependency> <groupId>org.apache.kylin</groupId> @@ -183,11 +183,11 @@ <shadedArtifactAttached>true</shadedArtifactAttached> <shadedClassifierName>job</shadedClassifierName> <artifactSet> - <excludes> - <exclude>io.netty:*</exclude> - <exclude>org.apache.zookeeper:*</exclude> - <exclude>net.sf.ehcache:*</exclude> - </excludes> + <excludes> + <exclude>io.netty:*</exclude> + <exclude>org.apache.zookeeper:*</exclude> + <exclude>net.sf.ehcache:*</exclude> + </excludes> </artifactSet> <filters> <filter> @@ -207,4 +207,18 @@ </build> + <profiles> + <profile> + <id>sandbox</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + <build> + <plugins> + + </plugins> + </build> + </profile> + </profiles> + </project> http://git-wip-us.apache.org/repos/asf/kylin/blob/1428bbc4/assembly/src/test/java/org/apache/kylin/RealizationRegistryTest.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/RealizationRegistryTest.java b/assembly/src/test/java/org/apache/kylin/RealizationRegistryTest.java new file mode 100644 index 0000000..b5aa8d0 --- /dev/null +++ b/assembly/src/test/java/org/apache/kylin/RealizationRegistryTest.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Set; + +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.util.LocalFileMetadataTestCase; +import org.apache.kylin.metadata.realization.RealizationRegistry; +import org.apache.kylin.metadata.realization.RealizationType; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + */ +public class RealizationRegistryTest extends LocalFileMetadataTestCase { + @Before + public void setup() throws Exception { + + createTestMetadata(); + } + + @After + public void after() throws Exception { + cleanupTestMetadata(); + } + + @Test + public void test() throws Exception { + final RealizationRegistry registry = RealizationRegistry.getInstance(KylinConfig.getInstanceFromEnv()); + final Set<RealizationType> realizationTypes = registry.getRealizationTypes(); + assertEquals(RealizationType.values().length, realizationTypes.size()); + for (RealizationType type : RealizationType.values()) { + assertTrue(realizationTypes.contains(type)); + } + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/1428bbc4/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java b/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java deleted file mode 100644 index f5b9741..0000000 --- a/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.engine.spark; - -import static org.junit.Assert.assertEquals; - -import java.io.File; -import java.text.SimpleDateFormat; -import java.util.List; -import java.util.TimeZone; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.AbstractKylinTestCase; -import org.apache.kylin.common.util.ClassUtil; -import org.apache.kylin.cube.CubeInstance; -import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.cube.CubeSegment; -import org.apache.kylin.cube.CubeUpdate; -import org.apache.kylin.job.DeployUtil; -import org.apache.kylin.job.engine.JobEngineConfig; -import org.apache.kylin.job.execution.AbstractExecutable; -import org.apache.kylin.job.execution.DefaultChainedExecutable; -import org.apache.kylin.job.execution.ExecutableState; -import org.apache.kylin.job.impl.threadpool.DefaultScheduler; -import org.apache.kylin.job.lock.MockJobLock; -import org.apache.kylin.job.manager.ExecutableManager; -import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -import com.google.common.collect.Lists; - -public class BuildCubeWithSparkTest { - - private CubeManager cubeManager; - private DefaultScheduler scheduler; - protected ExecutableManager jobService; - - private static final Log logger = LogFactory.getLog(BuildCubeWithSparkTest.class); - - protected void waitForJob(String jobId) { - while (true) { - AbstractExecutable job = jobService.getJob(jobId); - if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) { - break; - } else { - try { - Thread.sleep(5000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - } - } - - @BeforeClass - public static void beforeClass() throws Exception { - logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath()); - ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath()); - System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox"); - if (System.getProperty("hdp.version") == null) { - throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2"); - } - - } - - @Before - public void before() throws Exception { - HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA); - - DeployUtil.initCliWorkDir(); - DeployUtil.deployMetadata(); - DeployUtil.overrideJobJarLocations(); - - final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); - jobService = ExecutableManager.getInstance(kylinConfig); - for (String jobId : jobService.getAllJobIds()) { - jobService.deleteJob(jobId); - } - scheduler = DefaultScheduler.getInstance(); - scheduler.init(new JobEngineConfig(kylinConfig), new MockJobLock()); - if (!scheduler.hasStarted()) { - throw new RuntimeException("scheduler has not been started"); - } - cubeManager = CubeManager.getInstance(kylinConfig); - - } - - @After - public void after() { - HBaseMetadataTestCase.staticCleanupTestMetadata(); - } - - @Test - public void test() throws Exception { - final CubeSegment segment = createSegment(); - String confPath = new File(AbstractKylinTestCase.SANDBOX_TEST_DATA).getAbsolutePath(); - KylinConfig.getInstanceFromEnv().getCoprocessorLocalJar(); - String coprocessor = KylinConfig.getInstanceFromEnv().getCoprocessorLocalJar(); - logger.info("confPath location:" + confPath); - logger.info("coprocessor location:" + coprocessor); - final DefaultChainedExecutable cubingJob = new SparkBatchCubingEngine(confPath, coprocessor).createBatchCubingJob(segment, "BuildCubeWithSpark"); - jobService.addJob(cubingJob); - waitForJob(cubingJob.getId()); - assertEquals(ExecutableState.SUCCEED, jobService.getOutput(cubingJob.getId()).getState()); - } - - private void clearSegment(String cubeName) throws Exception { - CubeInstance cube = cubeManager.getCube(cubeName); - // remove all existing segments - CubeUpdate cubeBuilder = new CubeUpdate(cube); - cubeBuilder.setToRemoveSegs(cube.getSegments().toArray(new CubeSegment[cube.getSegments().size()])); - cubeManager.updateCube(cubeBuilder); - } - - private CubeSegment createSegment() throws Exception { - String cubeName = "test_kylin_cube_with_slr_left_join_empty"; - clearSegment(cubeName); - - SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); - f.setTimeZone(TimeZone.getTimeZone("GMT")); - long dateStart = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart(); - long dateEnd = f.parse("2050-11-12").getTime(); - - // this cube's start date is 0, end date is 20501112000000 - List<String> result = Lists.newArrayList(); - return cubeManager.appendSegments(cubeManager.getCube(cubeName), dateEnd); - - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/1428bbc4/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java deleted file mode 100644 index 331e21d..0000000 --- a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java +++ /dev/null @@ -1,333 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.job; - -import static org.junit.Assert.assertEquals; - -import java.io.File; -import java.lang.reflect.Method; -import java.text.SimpleDateFormat; -import java.util.List; -import java.util.TimeZone; -import java.util.concurrent.Callable; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.util.ToolRunner; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.AbstractKylinTestCase; -import org.apache.kylin.common.util.ClassUtil; -import org.apache.kylin.cube.CubeInstance; -import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.cube.CubeSegment; -import org.apache.kylin.cube.CubeUpdate; -import org.apache.kylin.engine.EngineFactory; -import org.apache.kylin.engine.mr.CubingJob; -import org.apache.kylin.job.engine.JobEngineConfig; -import org.apache.kylin.job.execution.AbstractExecutable; -import org.apache.kylin.job.execution.DefaultChainedExecutable; -import org.apache.kylin.job.execution.ExecutableState; -import org.apache.kylin.job.impl.threadpool.DefaultScheduler; -import org.apache.kylin.job.manager.ExecutableManager; -import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase; -import org.apache.kylin.storage.hbase.util.StorageCleanupJob; -import org.apache.kylin.storage.hbase.util.ZookeeperJobLock; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -import com.google.common.collect.Lists; - -public class BuildCubeWithEngineTest { - - private CubeManager cubeManager; - private DefaultScheduler scheduler; - protected ExecutableManager jobService; - private static boolean fastBuildMode = false; - - private static final Log logger = LogFactory.getLog(BuildCubeWithEngineTest.class); - - protected void waitForJob(String jobId) { - while (true) { - AbstractExecutable job = jobService.getJob(jobId); - if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) { - break; - } else { - try { - Thread.sleep(5000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - } - } - - @BeforeClass - public static void beforeClass() throws Exception { - logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath()); - ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath()); - - String fastModeStr = System.getProperty("fastBuildMode"); - if (fastModeStr != null && fastModeStr.equalsIgnoreCase("true")) { - fastBuildMode = true; - logger.info("Will use fast build mode"); - } else { - logger.info("Will not use fast build mode"); - } - - System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox"); - if (System.getProperty("hdp.version") == null) { - throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2"); - } - - } - - @Before - public void before() throws Exception { - HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA); - - DeployUtil.initCliWorkDir(); - DeployUtil.deployMetadata(); - DeployUtil.overrideJobJarLocations(); - - final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); - jobService = ExecutableManager.getInstance(kylinConfig); - scheduler = DefaultScheduler.getInstance(); - scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock()); - if (!scheduler.hasStarted()) { - throw new RuntimeException("scheduler has not been started"); - } - cubeManager = CubeManager.getInstance(kylinConfig); - for (String jobId : jobService.getAllJobIds()) { - if (jobService.getJob(jobId) instanceof CubingJob) { - jobService.deleteJob(jobId); - } - } - - } - - @After - public void after() { - HBaseMetadataTestCase.staticCleanupTestMetadata(); - } - - @Test - public void test() throws Exception { - DeployUtil.prepareTestDataForNormalCubes("test_kylin_cube_with_slr_left_join_empty"); - testInner(); - testLeft(); - } - - private void testInner() throws Exception { - String[] testCase = new String[] { "testInnerJoinCubeWithoutSlr", "testInnerJoinCubeWithSlr" }; - runTestAndAssertSucceed(testCase); - } - - private void testLeft() throws Exception { - String[] testCase = new String[] { "testLeftJoinCubeWithSlr", "testLeftJoinCubeWithoutSlr" }; - runTestAndAssertSucceed(testCase); - } - - private void runTestAndAssertSucceed(String[] testCase) throws Exception { - ExecutorService executorService = Executors.newFixedThreadPool(testCase.length); - final CountDownLatch countDownLatch = new CountDownLatch(testCase.length); - List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length); - for (int i = 0; i < testCase.length; i++) { - tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch))); - } - countDownLatch.await(); - try { - for (int i = 0; i < tasks.size(); ++i) { - Future<List<String>> task = tasks.get(i); - final List<String> jobIds = task.get(); - for (String jobId : jobIds) { - assertJobSucceed(jobId); - } - } - } catch (Exception ex) { - logger.error(ex); - throw ex; - } - } - - private void assertJobSucceed(String jobId) { - assertEquals("The job '" + jobId + "' is failed.", ExecutableState.SUCCEED, jobService.getOutput(jobId).getState()); - } - - private class TestCallable implements Callable<List<String>> { - - private final String methodName; - private final CountDownLatch countDownLatch; - - public TestCallable(String methodName, CountDownLatch countDownLatch) { - this.methodName = methodName; - this.countDownLatch = countDownLatch; - } - - @SuppressWarnings("unchecked") - @Override - public List<String> call() throws Exception { - try { - final Method method = BuildCubeWithEngineTest.class.getDeclaredMethod(methodName); - method.setAccessible(true); - return (List<String>) method.invoke(BuildCubeWithEngineTest.this); - } catch (Exception e) { - logger.error(e.getMessage()); - throw e; - } finally { - countDownLatch.countDown(); - } - } - } - - @SuppressWarnings("unused") - // called by reflection - private List<String> testInnerJoinCubeWithSlr() throws Exception { - clearSegment("test_kylin_cube_with_slr_empty"); - SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); - f.setTimeZone(TimeZone.getTimeZone("GMT")); - long date1 = 0; - long date2 = f.parse("2015-01-01").getTime(); - long date3 = f.parse("2022-01-01").getTime(); - List<String> result = Lists.newArrayList(); - - if (fastBuildMode) { - result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date3)); - } else { - result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date2)); - result.add(buildSegment("test_kylin_cube_with_slr_empty", date2, date3));//empty segment - } - return result; - } - - @SuppressWarnings("unused") - // called by reflection - private List<String> testInnerJoinCubeWithoutSlr() throws Exception { - - clearSegment("test_kylin_cube_without_slr_empty"); - SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); - f.setTimeZone(TimeZone.getTimeZone("GMT")); - long date1 = 0; - long date2 = f.parse("2013-01-01").getTime(); - long date3 = f.parse("2013-07-01").getTime(); - long date4 = f.parse("2022-01-01").getTime(); - List<String> result = Lists.newArrayList(); - - if (fastBuildMode) { - result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date4)); - } else { - result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date2)); - result.add(buildSegment("test_kylin_cube_without_slr_empty", date2, date3)); - result.add(buildSegment("test_kylin_cube_without_slr_empty", date3, date4)); - result.add(mergeSegment("test_kylin_cube_without_slr_empty", date1, date3));//don't merge all segments - } - return result; - - } - - @SuppressWarnings("unused") - // called by reflection - private List<String> testLeftJoinCubeWithoutSlr() throws Exception { - SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); - f.setTimeZone(TimeZone.getTimeZone("GMT")); - List<String> result = Lists.newArrayList(); - final String cubeName = "test_kylin_cube_without_slr_left_join_empty"; - clearSegment(cubeName); - - long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart(); - long date2 = f.parse("2012-06-01").getTime(); - long date3 = f.parse("2022-01-01").getTime(); - long date4 = f.parse("2023-01-01").getTime(); - - if (fastBuildMode) { - result.add(buildSegment("test_kylin_cube_without_slr_left_join_empty", date1, date4)); - } else { - result.add(buildSegment("test_kylin_cube_without_slr_left_join_empty", date1, date2)); - result.add(buildSegment("test_kylin_cube_without_slr_left_join_empty", date2, date3)); - result.add(buildSegment("test_kylin_cube_without_slr_left_join_empty", date3, date4));//empty segment - result.add(mergeSegment("test_kylin_cube_without_slr_left_join_empty", date1, date3));//don't merge all segments - } - - return result; - - } - - @SuppressWarnings("unused") - // called by reflection - private List<String> testLeftJoinCubeWithSlr() throws Exception { - String cubeName = "test_kylin_cube_with_slr_left_join_empty"; - clearSegment(cubeName); - - SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); - f.setTimeZone(TimeZone.getTimeZone("GMT")); - long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart(); - long date2 = f.parse("2013-01-01").getTime(); - long date3 = f.parse("2013-07-01").getTime(); - long date4 = f.parse("2022-01-01").getTime(); - - List<String> result = Lists.newArrayList(); - if (fastBuildMode) { - result.add(buildSegment(cubeName, date1, date4)); - } else { - result.add(buildSegment(cubeName, date1, date2)); - result.add(buildSegment(cubeName, date2, date3)); - result.add(buildSegment(cubeName, date3, date4)); - result.add(mergeSegment(cubeName, date1, date3));//don't merge all segments - } - return result; - - } - - private void clearSegment(String cubeName) throws Exception { - CubeInstance cube = cubeManager.getCube(cubeName); - // remove all existing segments - CubeUpdate cubeBuilder = new CubeUpdate(cube); - cubeBuilder.setToRemoveSegs(cube.getSegments().toArray(new CubeSegment[cube.getSegments().size()])); - cubeManager.updateCube(cubeBuilder); - } - - private String mergeSegment(String cubeName, long startDate, long endDate) throws Exception { - CubeSegment segment = cubeManager.mergeSegments(cubeManager.getCube(cubeName), startDate, endDate, true); - DefaultChainedExecutable job = EngineFactory.createBatchMergeJob(segment, "TEST"); - jobService.addJob(job); - waitForJob(job.getId()); - return job.getId(); - } - - private String buildSegment(String cubeName, long startDate, long endDate) throws Exception { - CubeSegment segment = cubeManager.appendSegments(cubeManager.getCube(cubeName), endDate); - DefaultChainedExecutable job = EngineFactory.createBatchCubingJob(segment, "TEST"); - jobService.addJob(job); - waitForJob(job.getId()); - return job.getId(); - } - - private int cleanupOldStorage() throws Exception { - String[] args = { "--delete", "true" }; - - int exitCode = ToolRunner.run(new StorageCleanupJob(), args); - return exitCode; - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/1428bbc4/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java deleted file mode 100644 index 9e60622..0000000 --- a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * - * contributor license agreements. See the NOTICE file distributed with - * - * this work for additional information regarding copyright ownership. - * - * The ASF licenses this file to You under the Apache License, Version 2.0 - * - * (the "License"); you may not use this file except in compliance with - * - * the License. You may obtain a copy of the License at - * - * - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * - * - * Unless required by applicable law or agreed to in writing, software - * - * distributed under the License is distributed on an "AS IS" BASIS, - * - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and - * - * limitations under the License. - * - * / - */ - -package org.apache.kylin.job; - -import org.apache.hadoop.util.ToolRunner; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.AbstractKylinTestCase; -import org.apache.kylin.common.util.ClassUtil; -import org.apache.kylin.common.util.DateFormat; -import org.apache.kylin.engine.streaming.OneOffStreamingBuilder; -import org.apache.kylin.engine.streaming.StreamingConfig; -import org.apache.kylin.engine.streaming.StreamingManager; -import org.apache.kylin.source.kafka.KafkaConfigManager; -import org.apache.kylin.source.kafka.config.KafkaConfig; -import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase; -import org.apache.kylin.storage.hbase.util.StorageCleanupJob; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.util.UUID; - -/** - * for streaming cubing case "test_streaming_table" - */ -public class BuildCubeWithStreamTest { - - private static final Logger logger = LoggerFactory.getLogger(BuildCubeWithStreamTest.class); - private static final String streamingName = "test_streaming_table_cube"; - private static final long startTime = DateFormat.stringToMillis("2015-01-01 00:00:00"); - private static final long endTime = DateFormat.stringToMillis("2015-01-03 00:00:00"); - private static final long batchInterval = 16 * 60 * 60 * 1000;//16 hours - - private KylinConfig kylinConfig; - - @BeforeClass - public static void beforeClass() throws Exception { - logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath()); - ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath()); - System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox"); - if (System.getProperty("hdp.version") == null) { - throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2"); - } - - } - - @Before - public void before() throws Exception { - HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA); - DeployUtil.overrideJobJarLocations(); - - kylinConfig = KylinConfig.getInstanceFromEnv(); - - final StreamingConfig config = StreamingManager.getInstance(KylinConfig.getInstanceFromEnv()).getStreamingConfig(streamingName); - - //Use a random topic for kafka data stream - KafkaConfig streamingConfig = KafkaConfigManager.getInstance(kylinConfig).getKafkaConfig(streamingName); - streamingConfig.setTopic(UUID.randomUUID().toString()); - KafkaConfigManager.getInstance(kylinConfig).saveKafkaConfig(streamingConfig); - - DeployUtil.prepareTestDataForStreamingCube(startTime, endTime, config.getCubeName(), streamingConfig); - } - - @AfterClass - public static void afterClass() throws Exception { - HBaseMetadataTestCase.staticCleanupTestMetadata(); - } - - @Test - public void test() throws Exception { - logger.info("start time:" + startTime + " end time:" + endTime + " batch interval:" + batchInterval + " batch count:" + ((endTime - startTime) / batchInterval)); - for (long start = startTime; start < endTime; start += batchInterval) { - logger.info(String.format("build batch:{%d, %d}", start, start + batchInterval)); - new OneOffStreamingBuilder(streamingName, start, start + batchInterval).build().run(); - } - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/1428bbc4/assembly/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/assembly/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java deleted file mode 100644 index 0158fad..0000000 --- a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.job; - -import static org.junit.Assert.assertEquals; - -import java.io.File; -import java.lang.reflect.Method; -import java.text.SimpleDateFormat; -import java.util.List; -import java.util.TimeZone; -import java.util.concurrent.Callable; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.util.ToolRunner; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.AbstractKylinTestCase; -import org.apache.kylin.common.util.ClassUtil; -import org.apache.kylin.engine.mr.invertedindex.BatchIIJobBuilder; -import org.apache.kylin.invertedindex.IIInstance; -import org.apache.kylin.invertedindex.IIManager; -import org.apache.kylin.invertedindex.IISegment; -import org.apache.kylin.job.engine.JobEngineConfig; -import org.apache.kylin.job.execution.AbstractExecutable; -import org.apache.kylin.job.execution.ExecutableState; -import org.apache.kylin.job.impl.threadpool.DefaultScheduler; -import org.apache.kylin.engine.mr.invertedindex.IIJob; -import org.apache.kylin.job.manager.ExecutableManager; -import org.apache.kylin.metadata.realization.RealizationStatusEnum; -import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase; -import org.apache.kylin.storage.hbase.util.StorageCleanupJob; -import org.apache.kylin.storage.hbase.util.ZookeeperJobLock; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; - -import com.google.common.collect.Lists; - -/** - * @author shaoshi - */ -public class BuildIIWithEngineTest { - - private JobEngineConfig jobEngineConfig; - private IIManager iiManager; - - private DefaultScheduler scheduler; - protected ExecutableManager jobService; - - protected static final String[] TEST_II_INSTANCES = new String[] { "test_kylin_ii_inner_join", "test_kylin_ii_left_join" }; - - private static final Log logger = LogFactory.getLog(BuildIIWithEngineTest.class); - - protected void waitForJob(String jobId) { - while (true) { - AbstractExecutable job = jobService.getJob(jobId); - if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) { - break; - } else { - try { - Thread.sleep(5000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - } - } - - @BeforeClass - public static void beforeClass() throws Exception { - logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath()); - ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath()); - System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox"); - if (System.getProperty("hdp.version") == null) { - throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2"); - } - } - - @Before - public void before() throws Exception { - HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA); - - //DeployUtil.initCliWorkDir(); - // DeployUtil.deployMetadata(); - DeployUtil.overrideJobJarLocations(); - - final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); - jobService = ExecutableManager.getInstance(kylinConfig); - scheduler = DefaultScheduler.getInstance(); - scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock()); - if (!scheduler.hasStarted()) { - throw new RuntimeException("scheduler has not been started"); - } - jobEngineConfig = new JobEngineConfig(kylinConfig); - for (String jobId : jobService.getAllJobIds()) { - if (jobService.getJob(jobId) instanceof IIJob) { - jobService.deleteJob(jobId); - } - } - - iiManager = IIManager.getInstance(kylinConfig); - for (String iiInstance : TEST_II_INSTANCES) { - - IIInstance ii = iiManager.getII(iiInstance); - if (ii.getStatus() != RealizationStatusEnum.DISABLED) { - ii.setStatus(RealizationStatusEnum.DISABLED); - iiManager.updateII(ii); - } - } - } - - @After - public void after() throws Exception { - - for (String iiInstance : TEST_II_INSTANCES) { - IIInstance ii = iiManager.getII(iiInstance); - if (ii.getStatus() != RealizationStatusEnum.READY) { - ii.setStatus(RealizationStatusEnum.READY); - iiManager.updateII(ii); - } - } - } - - @Test - public void testBuildII() throws Exception { - - String[] testCase = new String[] { "buildIIInnerJoin", "buildIILeftJoin" }; - ExecutorService executorService = Executors.newFixedThreadPool(testCase.length); - final CountDownLatch countDownLatch = new CountDownLatch(testCase.length); - List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length); - for (int i = 0; i < testCase.length; i++) { - tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch))); - } - countDownLatch.await(); - for (int i = 0; i < tasks.size(); ++i) { - Future<List<String>> task = tasks.get(i); - final List<String> jobIds = task.get(); - for (String jobId : jobIds) { - assertJobSucceed(jobId); - } - } - - } - - private void assertJobSucceed(String jobId) { - assertEquals(ExecutableState.SUCCEED, jobService.getOutput(jobId).getState()); - } - - private class TestCallable implements Callable<List<String>> { - - private final String methodName; - private final CountDownLatch countDownLatch; - - public TestCallable(String methodName, CountDownLatch countDownLatch) { - this.methodName = methodName; - this.countDownLatch = countDownLatch; - } - - @SuppressWarnings("unchecked") - @Override - public List<String> call() throws Exception { - try { - final Method method = BuildIIWithEngineTest.class.getDeclaredMethod(methodName); - method.setAccessible(true); - return (List<String>) method.invoke(BuildIIWithEngineTest.this); - } finally { - countDownLatch.countDown(); - } - } - } - - protected List<String> buildIIInnerJoin() throws Exception { - return buildII(TEST_II_INSTANCES[0]); - } - - protected List<String> buildIILeftJoin() throws Exception { - return buildII(TEST_II_INSTANCES[1]); - } - - protected List<String> buildII(String iiName) throws Exception { - clearSegment(iiName); - - SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); - f.setTimeZone(TimeZone.getTimeZone("GMT")); - - long date1 = 0; - long date2 = f.parse("2015-01-01").getTime(); - - List<String> result = Lists.newArrayList(); - result.add(buildSegment(iiName, date1, date2)); - return result; - } - - private void clearSegment(String iiName) throws Exception { - IIInstance ii = iiManager.getII(iiName); - ii.getSegments().clear(); - iiManager.updateII(ii); - } - - private String buildSegment(String iiName, long startDate, long endDate) throws Exception { - IIInstance iiInstance = iiManager.getII(iiName); - IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate); - iiInstance.getSegments().add(segment); - iiManager.updateII(iiInstance); - - BatchIIJobBuilder batchIIJobBuilder = new BatchIIJobBuilder(segment, "SYSTEM"); - IIJob job = batchIIJobBuilder.build(); - jobService.addJob(job); - waitForJob(job.getId()); - return job.getId(); - } - - private int cleanupOldStorage() throws Exception { - String[] args = { "--delete", "true" }; - - int exitCode = ToolRunner.run(new StorageCleanupJob(), args); - return exitCode; - } - - public static void main(String[] args) throws Exception { - BuildIIWithEngineTest instance = new BuildIIWithEngineTest(); - - BuildIIWithEngineTest.beforeClass(); - instance.before(); - instance.testBuildII(); - instance.after(); - - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/1428bbc4/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java deleted file mode 100644 index 2317e62..0000000 --- a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java +++ /dev/null @@ -1,290 +0,0 @@ -/* - * - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * - * contributor license agreements. See the NOTICE file distributed with - * - * this work for additional information regarding copyright ownership. - * - * The ASF licenses this file to You under the Apache License, Version 2.0 - * - * (the "License"); you may not use this file except in compliance with - * - * the License. You may obtain a copy of the License at - * - * - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * - * - * Unless required by applicable law or agreed to in writing, software - * - * distributed under the License is distributed on an "AS IS" BASIS, - * - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and - * - * limitations under the License. - * - * / - */ - -package org.apache.kylin.job; - -import static org.junit.Assert.fail; - -import java.io.File; -import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.*; - -import org.apache.hadoop.hbase.client.HTableInterface; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.util.ToolRunner; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.AbstractKylinTestCase; -import org.apache.kylin.common.util.ClassUtil; -import org.apache.kylin.common.util.DateFormat; -import org.apache.kylin.common.util.Pair; -import org.apache.kylin.engine.mr.JobBuilderSupport; -import org.apache.kylin.common.util.StreamingBatch; -import org.apache.kylin.common.util.StreamingMessage; -import org.apache.kylin.invertedindex.IIInstance; -import org.apache.kylin.invertedindex.IIManager; -import org.apache.kylin.invertedindex.IISegment; -import org.apache.kylin.invertedindex.index.Slice; -import org.apache.kylin.invertedindex.model.IIDesc; -import org.apache.kylin.invertedindex.model.IIJoinedFlatTableDesc; -import org.apache.kylin.invertedindex.model.IIKeyValueCodec; -import org.apache.kylin.invertedindex.model.IIRow; -import org.apache.kylin.invertedindex.index.SliceBuilder; -import org.apache.kylin.job.common.ShellExecutable; -import org.apache.kylin.job.constant.ExecutableConstants; -import org.apache.kylin.job.engine.JobEngineConfig; -import org.apache.kylin.storage.hbase.ii.IICreateHTableJob; -import org.apache.kylin.metadata.model.TblColRef; -import org.apache.kylin.metadata.realization.RealizationStatusEnum; -import org.apache.kylin.source.hive.HiveTableReader; -import org.apache.kylin.storage.hbase.HBaseConnection; -import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase; -import org.apache.kylin.storage.hbase.util.StorageCleanupJob; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Lists; - -/** - */ -public class BuildIIWithStreamTest { - - private static final Logger logger = LoggerFactory.getLogger(BuildIIWithStreamTest.class); - - private static final String[] II_NAME = new String[] { "test_kylin_ii_left_join", "test_kylin_ii_inner_join" }; - private IIManager iiManager; - private KylinConfig kylinConfig; - - @BeforeClass - public static void beforeClass() throws Exception { - logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath()); - ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath()); - if (System.getProperty("hdp.version") == null) { - throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2"); - } - } - - @Before - public void before() throws Exception { - HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA); - DeployUtil.overrideJobJarLocations(); - - kylinConfig = KylinConfig.getInstanceFromEnv(); - iiManager = IIManager.getInstance(kylinConfig); - for (String iiInstance : II_NAME) { - - IIInstance ii = iiManager.getII(iiInstance); - if (ii.getStatus() != RealizationStatusEnum.DISABLED) { - ii.setStatus(RealizationStatusEnum.DISABLED); - iiManager.updateII(ii); - } - } - } - - @AfterClass - public static void cleanup() throws Exception { - cleanupOldStorage(); - } - - private String createIntermediateTable(IIDesc desc, KylinConfig kylinConfig) throws IOException { - IIJoinedFlatTableDesc intermediateTableDesc = new IIJoinedFlatTableDesc(desc); - JobEngineConfig jobEngineConfig = new JobEngineConfig(kylinConfig); - final String uuid = UUID.randomUUID().toString(); - final String useDatabaseHql = "USE " + kylinConfig.getHiveDatabaseForIntermediateTable() + ";"; - final String dropTableHql = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc); - final String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, JobBuilderSupport.getJobWorkingDir(jobEngineConfig, uuid)); - String insertDataHqls; - try { - insertDataHqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, jobEngineConfig); - } catch (IOException e1) { - e1.printStackTrace(); - throw new RuntimeException("Failed to generate insert data SQL for intermediate table."); - } - - ShellExecutable step = new ShellExecutable(); - StringBuffer buf = new StringBuffer(); - buf.append("hive -e \""); - buf.append(useDatabaseHql + "\n"); - buf.append(dropTableHql + "\n"); - buf.append(createTableHql + "\n"); - buf.append(insertDataHqls + "\n"); - buf.append("\""); - - step.setCmd(buf.toString()); - logger.info(step.getCmd()); - step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE); - kylinConfig.getCliCommandExecutor().execute(step.getCmd(), null); - return intermediateTableDesc.getTableName(); - } - - private void clearSegment(String iiName) throws Exception { - IIInstance ii = iiManager.getII(iiName); - ii.getSegments().clear(); - iiManager.updateII(ii); - } - - private IISegment createSegment(String iiName) throws Exception { - clearSegment(iiName); - SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); - f.setTimeZone(TimeZone.getTimeZone("GMT")); - - long date1 = 0; - long date2 = f.parse("2015-01-01").getTime(); - return buildSegment(iiName, date1, date2); - } - - private IISegment buildSegment(String iiName, long startDate, long endDate) throws Exception { - IIInstance iiInstance = iiManager.getII(iiName); - IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate); - iiInstance.getSegments().add(segment); - iiManager.updateII(iiInstance); - return segment; - } - - private void buildII(String iiName) throws Exception { - final IIDesc desc = iiManager.getII(iiName).getDescriptor(); - final String tableName = createIntermediateTable(desc, kylinConfig); - logger.info("intermediate table name:" + tableName); - - HiveTableReader reader = new HiveTableReader("default", tableName); - final List<TblColRef> tblColRefs = desc.listAllColumns(); - for (TblColRef tblColRef : tblColRefs) { - if (desc.isMetricsCol(tblColRef)) { - logger.info("matrix:" + tblColRef.getName()); - } else { - logger.info("measure:" + tblColRef.getName()); - } - } - final IISegment segment = createSegment(iiName); - final HTableInterface htable = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(segment.getStorageLocationIdentifier()); - String[] args = new String[] { "-iiname", iiName, "-htablename", segment.getStorageLocationIdentifier() }; - ToolRunner.run(new IICreateHTableJob(), args); - - final IIDesc iiDesc = segment.getIIDesc(); - final SliceBuilder sliceBuilder = new SliceBuilder(desc, (short) 0); - - List<String[]> sorted = getSortedRows(reader, desc.getTimestampColumn()); - int count = sorted.size(); - ArrayList<StreamingMessage> messages = Lists.newArrayList(); - for (String[] row : sorted) { - messages.add((parse(row))); - if (messages.size() >= iiDesc.getSliceSize()) { - build(sliceBuilder, new StreamingBatch(messages, Pair.newPair(System.currentTimeMillis(), System.currentTimeMillis())), htable); - messages.clear(); - } - } - - if (!messages.isEmpty()) { - build(sliceBuilder, new StreamingBatch(messages, Pair.newPair(System.currentTimeMillis(), System.currentTimeMillis())), htable); - } - - reader.close(); - logger.info("total record count:" + count + " htable:" + segment.getStorageLocationIdentifier()); - logger.info("stream build finished, htable name:" + segment.getStorageLocationIdentifier()); - } - - @Test - public void test() throws Exception { - for (String iiName : II_NAME) { - buildII(iiName); - IIInstance ii = iiManager.getII(iiName); - if (ii.getStatus() != RealizationStatusEnum.READY) { - ii.setStatus(RealizationStatusEnum.READY); - iiManager.updateII(ii); - } - } - } - - private void build(SliceBuilder sliceBuilder, StreamingBatch batch, HTableInterface htable) throws IOException { - final Slice slice = sliceBuilder.buildSlice(batch); - try { - loadToHBase(htable, slice, new IIKeyValueCodec(slice.getInfo())); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private void loadToHBase(HTableInterface hTable, Slice slice, IIKeyValueCodec codec) throws IOException { - List<Put> data = Lists.newArrayList(); - for (IIRow row : codec.encodeKeyValue(slice)) { - final byte[] key = row.getKey().get(); - final byte[] value = row.getValue().get(); - Put put = new Put(key); - put.add(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_QUALIFIER_BYTES, value); - final ImmutableBytesWritable dictionary = row.getDictionary(); - final byte[] dictBytes = dictionary.get(); - if (dictionary.getOffset() == 0 && dictionary.getLength() == dictBytes.length) { - put.add(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_DICTIONARY_BYTES, dictBytes); - } else { - throw new RuntimeException("dict offset should be 0, and dict length should be " + dictBytes.length + " but they are" + dictionary.getOffset() + " " + dictionary.getLength()); - } - data.add(put); - } - hTable.put(data); - //omit hTable.flushCommits(), because htable is auto flush - } - - private StreamingMessage parse(String[] row) { - return new StreamingMessage(Lists.newArrayList(row), System.currentTimeMillis(), System.currentTimeMillis(), Collections.<String, Object>emptyMap()); - } - - private List<String[]> getSortedRows(HiveTableReader reader, final int tsCol) throws IOException { - List<String[]> unsorted = Lists.newArrayList(); - while (reader.next()) { - unsorted.add(reader.getRow()); - } - Collections.sort(unsorted, new Comparator<String[]>() { - @Override - public int compare(String[] o1, String[] o2) { - long t1 = DateFormat.stringToMillis(o1[tsCol]); - long t2 = DateFormat.stringToMillis(o2[tsCol]); - return Long.compare(t1, t2); - } - }); - return unsorted; - } - - private static int cleanupOldStorage() throws Exception { - String[] args = { "--delete", "true" }; - - int exitCode = ToolRunner.run(new StorageCleanupJob(), args); - return exitCode; - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/1428bbc4/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java b/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java index 92c9dae..1267ab7 100644 --- a/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java +++ b/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java @@ -20,12 +20,13 @@ package org.apache.kylin.job; import java.io.File; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.AbstractKylinTestCase; import org.apache.kylin.common.util.ClassUtil; -import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase; +import org.apache.kylin.common.util.HBaseMetadataTestCase; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; @@ -45,7 +46,7 @@ public class DeployLocalMetaToRemoteTest { logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath()); ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath()); System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox"); - if (System.getProperty("hdp.version") == null) { + if (StringUtils.isEmpty(System.getProperty("hdp.version"))) { throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2"); } }
