http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/tools/RowCounterCLI.java ---------------------------------------------------------------------- diff --git a/job/src/main/java/org/apache/kylin/job/tools/RowCounterCLI.java b/job/src/main/java/org/apache/kylin/job/tools/RowCounterCLI.java deleted file mode 100644 index 3329d27..0000000 --- a/job/src/main/java/org/apache/kylin/job/tools/RowCounterCLI.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kylin.job.tools; - -import java.io.IOException; -import java.util.Iterator; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.client.HConnection; -import org.apache.hadoop.hbase.client.HConnectionManager; -import org.apache.hadoop.hbase.client.HTableInterface; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.kylin.common.util.Bytes; -import org.apache.kylin.common.util.BytesUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - */ -public class RowCounterCLI { - private static final Logger logger = LoggerFactory.getLogger(RowCounterCLI.class); - - public static void main(String[] args) throws IOException { - - if (args == null || args.length != 3) { - System.out.println("Usage: hbase org.apache.hadoop.util.RunJar kylin-job-latest.jar org.apache.kylin.job.tools.RowCounterCLI [HTABLE_NAME] [STARTKEY] [ENDKEY]"); - } - - System.out.println(args[0]); - String htableName = args[0]; - System.out.println(args[1]); - byte[] startKey = BytesUtil.fromReadableText(args[1]); - System.out.println(args[2]); - byte[] endKey = BytesUtil.fromReadableText(args[2]); - - if (startKey == null) { - System.out.println("startkey is null "); - } else { - System.out.println("startkey lenght: " + startKey.length); - } - - System.out.println("start key in binary: " + Bytes.toStringBinary(startKey)); - System.out.println("end key in binary: " + Bytes.toStringBinary(endKey)); - - Configuration conf = HBaseConfiguration.create(); - - Scan scan = new Scan(); - scan.setCaching(1024); - scan.setCacheBlocks(true); - scan.setStartRow(startKey); - scan.setStopRow(endKey); - - logger.info("My Scan " + scan.toString()); - - HConnection conn = HConnectionManager.createConnection(conf); - HTableInterface tableInterface = conn.getTable(htableName); - - Iterator<Result> iterator = tableInterface.getScanner(scan).iterator(); - int counter = 0; - while (iterator.hasNext()) { - iterator.next(); - counter++; - if (counter % 1000 == 1) { - System.out.println("number of rows: " + counter); - } - } - System.out.println("number of rows: " + counter); - tableInterface.close(); - conn.close(); - } -}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/tools/UpdateHTableHostCLI.java ---------------------------------------------------------------------- diff --git a/job/src/main/java/org/apache/kylin/job/tools/UpdateHTableHostCLI.java b/job/src/main/java/org/apache/kylin/job/tools/UpdateHTableHostCLI.java deleted file mode 100644 index 30ffb46..0000000 --- a/job/src/main/java/org/apache/kylin/job/tools/UpdateHTableHostCLI.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.tools; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import org.apache.commons.lang.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.HadoopUtil; -import org.apache.kylin.cube.CubeInstance; -import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.cube.CubeSegment; -import org.apache.kylin.invertedindex.IIInstance; -import org.apache.kylin.invertedindex.IIManager; -import org.apache.kylin.invertedindex.IISegment; -import org.apache.kylin.metadata.model.SegmentStatusEnum; -import org.apache.kylin.metadata.realization.IRealizationConstants; -import org.apache.kylin.metadata.realization.RealizationStatusEnum; - -import com.google.common.collect.Lists; - -public class UpdateHTableHostCLI { - private static final Log logger = LogFactory.getLog(UpdateHTableHostCLI.class); - private List<String> updatedResources = Lists.newArrayList(); - private List<String> errorMsgs = Lists.newArrayList(); - - private List<String> htables; - private HBaseAdmin hbaseAdmin; - private KylinConfig kylinConfig; - private String oldHostValue; - - public UpdateHTableHostCLI(List<String> htables, String oldHostValue) throws IOException { - this.htables = htables; - this.oldHostValue = oldHostValue; - this.hbaseAdmin = new HBaseAdmin(HadoopUtil.getCurrentHBaseConfiguration()); - this.kylinConfig = KylinConfig.getInstanceFromEnv(); - } - - public static void main(String args[]) throws Exception { - if (args.length < 1) { - printUsageAndExit(); - } - - List<String> tableNames = getHTableNames(KylinConfig.getInstanceFromEnv()); - if (!args[0].toLowerCase().equals("-from")) { - printUsageAndExit(); - } - String oldHostValue = args[1].toLowerCase(); - String filterType = args[2].toLowerCase(); - if (filterType.equals("-table")) { - tableNames = filterByTables(tableNames, Arrays.asList(args).subList(3, args.length)); - } else if (filterType.equals("-cube")) { - tableNames = filterByCubes(tableNames, Arrays.asList(args).subList(3, args.length)); - } else if (!filterType.equals("-all")) { - printUsageAndExit(); - } - logger.info("These htables are needed to be updated: " + StringUtils.join(tableNames, ",")); - - UpdateHTableHostCLI updateHTableHostCLI = new UpdateHTableHostCLI(tableNames, oldHostValue); - updateHTableHostCLI.execute(); - - logger.info("================================================================="); - logger.info("Run UpdateHTableHostCLI completed;"); - - if (!updateHTableHostCLI.updatedResources.isEmpty()) { - logger.info("Following resources are updated successfully:"); - for (String s : updateHTableHostCLI.updatedResources) { - logger.info(s); - } - } else { - logger.warn("No resource updated."); - } - - if (!updateHTableHostCLI.errorMsgs.isEmpty()) { - logger.info("Here are the error/warning messages, you may need to check:"); - for (String s : updateHTableHostCLI.errorMsgs) { - logger.warn(s); - } - } else { - logger.info("No error or warning messages; The update succeeds."); - } - - logger.info("================================================================="); - } - - private static void printUsageAndExit() { - logger.info("Usage: exec -from oldHostValue -all|-cube cubeA,cubeB|-table tableA,tableB"); - System.exit(0); - } - - private static List<String> getHTableNames(KylinConfig config) { - CubeManager cubeMgr = CubeManager.getInstance(config); - - ArrayList<String> result = new ArrayList<String>(); - for (CubeInstance cube : cubeMgr.listAllCubes()) { - for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) { - String tableName = seg.getStorageLocationIdentifier(); - if (!StringUtils.isBlank(tableName)) { - result.add(tableName); - System.out.println("added new table: " + tableName); - } - } - } - - for (IIInstance ii : IIManager.getInstance(config).listAllIIs()) { - if (ii.getStatus() == RealizationStatusEnum.READY) { - for (IISegment seg : ii.getSegments()) {//streaming segment is never "READY" - String tableName = seg.getStorageLocationIdentifier(); - if (!StringUtils.isBlank(tableName)) { - result.add(tableName); - System.out.println("added new table: " + tableName); - } - } - } - } - - return result; - } - - private static List<String> filterByCubes(List<String> allTableNames, List<String> cubeNames) { - CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); - List<String> result = Lists.newArrayList(); - for (String c : cubeNames) { - c = c.trim(); - if (c.endsWith(",")) - c = c.substring(0, c.length() - 1); - - CubeInstance cubeInstance = cubeManager.getCube(c); - for (CubeSegment segment : cubeInstance.getSegments()) { - String tableName = segment.getStorageLocationIdentifier(); - if (allTableNames.contains(tableName)) { - result.add(tableName); - } - } - } - return result; - } - - private static List<String> filterByTables(List<String> allTableNames, List<String> tableNames) { - List<String> result = Lists.newArrayList(); - for (String t : tableNames) { - t = t.trim(); - if (t.endsWith(",")) - t = t.substring(0, t.length() - 1); - - if (allTableNames.contains(t)) { - result.add(t); - } - } - return result; - } - - private void updateHtable(String tableName) throws IOException { - HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName)); - if (oldHostValue.equals(desc.getValue(IRealizationConstants.HTableTag))) { - desc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix()); - hbaseAdmin.disableTable(tableName); - hbaseAdmin.modifyTable(tableName, desc); - hbaseAdmin.enableTable(tableName); - - updatedResources.add(tableName); - } - } - - public void execute() { - for (String htable : htables) { - try { - updateHtable(htable); - } catch (IOException ex) { - ex.printStackTrace(); - errorMsgs.add("Update HTable[" + htable + "] failed: " + ex.getMessage()); - } - } - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/BaseTestExecutable.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/BaseTestExecutable.java b/job/src/test/java/org/apache/kylin/job/BaseTestExecutable.java deleted file mode 100644 index 2255b09..0000000 --- a/job/src/test/java/org/apache/kylin/job/BaseTestExecutable.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job; - -import org.apache.kylin.job.execution.AbstractExecutable; - -/** - * Created by qianzhou on 12/16/14. - */ -public abstract class BaseTestExecutable extends AbstractExecutable { - - public BaseTestExecutable() { - super(); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java deleted file mode 100644 index e8162fe..0000000 --- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java +++ /dev/null @@ -1,286 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job; - -import static org.junit.Assert.assertEquals; - -import java.io.File; -import java.lang.reflect.Method; -import java.text.SimpleDateFormat; -import java.util.List; -import java.util.TimeZone; -import java.util.concurrent.Callable; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.lock.ZookeeperJobLock; -import org.apache.kylin.common.util.AbstractKylinTestCase; -import org.apache.kylin.common.util.ClassUtil; -import org.apache.kylin.common.util.HBaseMetadataTestCase; -import org.apache.kylin.cube.CubeInstance; -import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.cube.CubeSegment; -import org.apache.kylin.job.cube.CubingJob; -import org.apache.kylin.job.cube.CubingJobBuilder; -import org.apache.kylin.job.engine.JobEngineConfig; -import org.apache.kylin.job.execution.AbstractExecutable; -import org.apache.kylin.job.execution.ExecutableState; -import org.apache.kylin.job.impl.threadpool.DefaultScheduler; -import org.apache.kylin.job.manager.ExecutableManager; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -import com.google.common.collect.Lists; - -public class BuildCubeWithEngineTest { - - private JobEngineConfig jobEngineConfig; - - private CubeManager cubeManager; - - private DefaultScheduler scheduler; - - protected ExecutableManager jobService; - - private static final Log logger = LogFactory.getLog(BuildCubeWithEngineTest.class); - - protected void waitForJob(String jobId) { - while (true) { - AbstractExecutable job = jobService.getJob(jobId); - if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) { - break; - } else { - try { - Thread.sleep(5000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - } - } - - @BeforeClass - public static void beforeClass() throws Exception { - logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath()); - ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath()); - System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox"); - if (System.getProperty("hdp.version") == null) { - throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2"); - } - } - - @Before - public void before() throws Exception { - HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA); - - DeployUtil.initCliWorkDir(); - DeployUtil.deployMetadata(); - DeployUtil.overrideJobJarLocations(); - - final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); - jobService = ExecutableManager.getInstance(kylinConfig); - scheduler = DefaultScheduler.getInstance(); - scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock()); - if (!scheduler.hasStarted()) { - throw new RuntimeException("scheduler has not been started"); - } - cubeManager = CubeManager.getInstance(kylinConfig); - jobEngineConfig = new JobEngineConfig(kylinConfig); - for (String jobId : jobService.getAllJobIds()) { - if (jobService.getJob(jobId) instanceof CubingJob) { - jobService.deleteJob(jobId); - } - } - - } - - @After - public void after() { - HBaseMetadataTestCase.staticCleanupTestMetadata(); - } - - @Test - public void test() throws Exception { - testInner(); - testLeft(); - } - - private void testInner() throws Exception { - DeployUtil.prepareTestData("inner", "test_kylin_cube_with_slr_empty"); - String[] testCase = new String[] { "testInnerJoinCube", "testInnerJoinCube2", }; - runTestAndAssertSucceed(testCase); - } - - private void testLeft() throws Exception { - DeployUtil.prepareTestData("left", "test_kylin_cube_with_slr_left_join_empty"); - String[] testCase = new String[] { "testLeftJoinCube", "testLeftJoinCube2", }; - runTestAndAssertSucceed(testCase); - } - - private void runTestAndAssertSucceed(String[] testCase) throws Exception { - ExecutorService executorService = Executors.newFixedThreadPool(testCase.length); - final CountDownLatch countDownLatch = new CountDownLatch(testCase.length); - List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length); - for (int i = 0; i < testCase.length; i++) { - tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch))); - } - countDownLatch.await(); - try { - for (int i = 0; i < tasks.size(); ++i) { - Future<List<String>> task = tasks.get(i); - final List<String> jobIds = task.get(); - for (String jobId : jobIds) { - assertJobSucceed(jobId); - } - } - } catch (Exception ex) { - logger.error(ex); - throw ex; - } - } - - private void assertJobSucceed(String jobId) { - assertEquals("The job '" + jobId + "' is failed.", ExecutableState.SUCCEED, jobService.getOutput(jobId).getState()); - } - - private class TestCallable implements Callable<List<String>> { - - private final String methodName; - private final CountDownLatch countDownLatch; - - public TestCallable(String methodName, CountDownLatch countDownLatch) { - this.methodName = methodName; - this.countDownLatch = countDownLatch; - } - - @SuppressWarnings("unchecked") - @Override - public List<String> call() throws Exception { - try { - final Method method = BuildCubeWithEngineTest.class.getDeclaredMethod(methodName); - method.setAccessible(true); - return (List<String>) method.invoke(BuildCubeWithEngineTest.this); - } finally { - countDownLatch.countDown(); - } - } - } - - @SuppressWarnings("unused") - // called by reflection - private List<String> testInnerJoinCube2() throws Exception { - clearSegment("test_kylin_cube_with_slr_empty"); - SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); - f.setTimeZone(TimeZone.getTimeZone("GMT")); - long date1 = 0; - long date2 = f.parse("2013-01-01").getTime(); - long date3 = f.parse("2022-01-01").getTime(); - List<String> result = Lists.newArrayList(); - result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date2)); - result.add(buildSegment("test_kylin_cube_with_slr_empty", date2, date3)); - - // empty segment - long date4 = f.parse("2050-01-01").getTime(); - result.add(buildSegment("test_kylin_cube_with_slr_empty", date3, date4)); - return result; - } - - @SuppressWarnings("unused") - // called by reflection - private List<String> testInnerJoinCube() throws Exception { - clearSegment("test_kylin_cube_without_slr_empty"); - - SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); - f.setTimeZone(TimeZone.getTimeZone("GMT")); - - // this cube's start date is 0, end date is 20501112000000 - long date1 = 0; - long date2 = f.parse("2022-01-01").getTime(); - - // this cube doesn't support incremental build, always do full build - - List<String> result = Lists.newArrayList(); - result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date2)); - return result; - } - - @SuppressWarnings("unused") - // called by reflection - private List<String> testLeftJoinCube2() throws Exception { - SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); - f.setTimeZone(TimeZone.getTimeZone("GMT")); - List<String> result = Lists.newArrayList(); - final String cubeName = "test_kylin_cube_without_slr_left_join_empty"; - // this cube's start date is 0, end date is 20120601000000 - long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart(); - long dateEnd = f.parse("2012-06-01").getTime(); - - clearSegment(cubeName); - result.add(buildSegment(cubeName, dateStart, dateEnd)); - - // then submit an append job, start date is 20120601000000, end - // date is 20220101000000 - dateStart = f.parse("2012-06-01").getTime(); - dateEnd = f.parse("2022-01-01").getTime(); - result.add(buildSegment(cubeName, dateStart, dateEnd)); - return result; - - } - - @SuppressWarnings("unused") - // called by reflection - private List<String> testLeftJoinCube() throws Exception { - String cubeName = "test_kylin_cube_with_slr_left_join_empty"; - clearSegment(cubeName); - - SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); - f.setTimeZone(TimeZone.getTimeZone("GMT")); - long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart(); - long dateEnd = f.parse("2050-11-12").getTime(); - - // this cube's start date is 0, end date is 20501112000000 - List<String> result = Lists.newArrayList(); - result.add(buildSegment(cubeName, dateStart, dateEnd)); - return result; - - } - - private void clearSegment(String cubeName) throws Exception { - CubeInstance cube = cubeManager.getCube(cubeName); - cube.getSegments().clear(); - cubeManager.updateCube(cube); - } - - private String buildSegment(String cubeName, long startDate, long endDate) throws Exception { - CubeSegment segment = cubeManager.appendSegments(cubeManager.getCube(cubeName), endDate); - CubingJobBuilder cubingJobBuilder = new CubingJobBuilder(jobEngineConfig); - CubingJob job = cubingJobBuilder.buildJob(segment); - jobService.addJob(job); - waitForJob(job.getId()); - return job.getId(); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java deleted file mode 100644 index fb9167b..0000000 --- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java +++ /dev/null @@ -1,255 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job; - -import static org.junit.Assert.assertEquals; - -import java.io.File; -import java.io.IOException; -import java.lang.reflect.Method; -import java.text.SimpleDateFormat; -import java.util.List; -import java.util.TimeZone; -import java.util.concurrent.Callable; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.util.ToolRunner; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.lock.ZookeeperJobLock; -import org.apache.kylin.common.util.AbstractKylinTestCase; -import org.apache.kylin.common.util.ClassUtil; -import org.apache.kylin.common.util.HBaseMetadataTestCase; -import org.apache.kylin.invertedindex.IIInstance; -import org.apache.kylin.invertedindex.IIManager; -import org.apache.kylin.invertedindex.IISegment; -import org.apache.kylin.job.engine.JobEngineConfig; -import org.apache.kylin.job.execution.AbstractExecutable; -import org.apache.kylin.job.execution.ExecutableState; -import org.apache.kylin.job.hadoop.cube.StorageCleanupJob; -import org.apache.kylin.job.impl.threadpool.DefaultScheduler; -import org.apache.kylin.job.invertedindex.IIJob; -import org.apache.kylin.job.invertedindex.IIJobBuilder; -import org.apache.kylin.job.manager.ExecutableManager; -import org.apache.kylin.metadata.realization.RealizationStatusEnum; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -import com.google.common.collect.Lists; - -/** - * - * @author shaoshi - * - */ -public class BuildIIWithEngineTest { - - private JobEngineConfig jobEngineConfig; - private IIManager iiManager; - - private DefaultScheduler scheduler; - protected ExecutableManager jobService; - - protected static final String TEST_II_NAME = "test_kylin_ii"; - private static final Log logger = LogFactory.getLog(BuildCubeWithEngineTest.class); - - protected void waitForJob(String jobId) { - while (true) { - AbstractExecutable job = jobService.getJob(jobId); - if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) { - break; - } else { - try { - Thread.sleep(5000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - } - } - - @BeforeClass - public static void beforeClass() throws Exception { - logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath()); - ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath()); - if (System.getProperty("hdp.version") == null) { - throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2"); - } - } - - @Before - public void before() throws Exception { - HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA); - - DeployUtil.initCliWorkDir(); - // DeployUtil.deployMetadata(); - DeployUtil.overrideJobJarLocations(); - - final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); - jobService = ExecutableManager.getInstance(kylinConfig); - scheduler = DefaultScheduler.getInstance(); - scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock()); - if (!scheduler.hasStarted()) { - throw new RuntimeException("scheduler has not been started"); - } - iiManager = IIManager.getInstance(kylinConfig); - jobEngineConfig = new JobEngineConfig(kylinConfig); - for (String jobId : jobService.getAllJobIds()) { - if (jobService.getJob(jobId) instanceof IIJob) { - jobService.deleteJob(jobId); - } - } - - IIInstance ii = iiManager.getII(TEST_II_NAME); - if (ii.getStatus() != RealizationStatusEnum.DISABLED) { - ii.setStatus(RealizationStatusEnum.DISABLED); - iiManager.updateII(ii); - } - } - - @After - public void after() throws Exception { - IIInstance ii = iiManager.getII(TEST_II_NAME); - if (ii.getStatus() != RealizationStatusEnum.READY) { - ii.setStatus(RealizationStatusEnum.READY); - iiManager.updateII(ii); - } - backup(); - } - - @Test - public void testBuildII() throws Exception { - - String[] testCase = new String[] { "buildII" }; - ExecutorService executorService = Executors.newFixedThreadPool(testCase.length); - final CountDownLatch countDownLatch = new CountDownLatch(testCase.length); - List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length); - for (int i = 0; i < testCase.length; i++) { - tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch))); - } - countDownLatch.await(); - for (int i = 0; i < tasks.size(); ++i) { - Future<List<String>> task = tasks.get(i); - final List<String> jobIds = task.get(); - for (String jobId : jobIds) { - assertJobSucceed(jobId); - } - } - - } - - private void assertJobSucceed(String jobId) { - assertEquals(ExecutableState.SUCCEED, jobService.getOutput(jobId).getState()); - } - - private class TestCallable implements Callable<List<String>> { - - private final String methodName; - private final CountDownLatch countDownLatch; - - public TestCallable(String methodName, CountDownLatch countDownLatch) { - this.methodName = methodName; - this.countDownLatch = countDownLatch; - } - - @SuppressWarnings("unchecked") - @Override - public List<String> call() throws Exception { - try { - final Method method = BuildIIWithEngineTest.class.getDeclaredMethod(methodName); - method.setAccessible(true); - return (List<String>) method.invoke(BuildIIWithEngineTest.this); - } finally { - countDownLatch.countDown(); - } - } - } - - protected List<String> buildII() throws Exception { - clearSegment(TEST_II_NAME); - - SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); - f.setTimeZone(TimeZone.getTimeZone("GMT")); - - // this cube's start date is 0, end date is 2015-1-1 - long date1 = 0; - long date2 = f.parse("2015-01-01").getTime(); - - // this cube doesn't support incremental build, always do full build - - List<String> result = Lists.newArrayList(); - result.add(buildSegment(TEST_II_NAME, date1, date2)); - return result; - } - - private void clearSegment(String iiName) throws Exception { - IIInstance ii = iiManager.getII(iiName); - ii.getSegments().clear(); - iiManager.updateII(ii); - } - - private String buildSegment(String iiName, long startDate, long endDate) throws Exception { - IIInstance iiInstance = iiManager.getII(iiName); - IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate); - iiInstance.getSegments().add(segment); - iiManager.updateII(iiInstance); - IIJobBuilder iiJobBuilder = new IIJobBuilder(jobEngineConfig); - IIJob job = iiJobBuilder.buildJob(segment); - jobService.addJob(job); - waitForJob(job.getId()); - return job.getId(); - } - - private int cleanupOldStorage() throws Exception { - String[] args = { "--delete", "true" }; - - int exitCode = ToolRunner.run(new StorageCleanupJob(), args); - return exitCode; - } - - private void backup() throws Exception { - int exitCode = cleanupOldStorage(); - if (exitCode == 0 && "true".equals(System.getProperty("kylin.exportHtables"))) { - exportHBaseData(); - } - } - - private void exportHBaseData() throws IOException { - ExportHBaseData export = new ExportHBaseData(); - export.exportTables(); - export.tearDown(); - } - - public static void main(String[] args) throws Exception { - BuildIIWithEngineTest instance = new BuildIIWithEngineTest(); - - BuildIIWithEngineTest.beforeClass(); - instance.before(); - instance.testBuildII(); - instance.after(); - - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/DataGenTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/DataGenTest.java b/job/src/test/java/org/apache/kylin/job/DataGenTest.java deleted file mode 100644 index 6cc02e2..0000000 --- a/job/src/test/java/org/apache/kylin/job/DataGenTest.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job; - -import static org.junit.Assert.assertTrue; - -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.job.dataGen.FactTableGenerator; -import org.apache.kylin.metadata.MetadataManager; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -/** - */ -public class DataGenTest extends LocalFileMetadataTestCase { - - @Before - public void before() throws Exception { - this.createTestMetadata(); - MetadataManager.clearCache(); - } - - @After - public void after() throws Exception { - this.cleanupTestMetadata(); - } - - @Test - public void testBasics() throws Exception { - String content = FactTableGenerator.generate("test_kylin_cube_with_slr_ready", "10000", "1", null, "inner");// default settings - System.out.println(content); - assertTrue(content.contains("FP-non GTC")); - assertTrue(content.contains("ABIN")); - - DeployUtil.overrideFactTableData(content, "default.test_kylin_fact"); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/DeployUtil.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/DeployUtil.java b/job/src/test/java/org/apache/kylin/job/DeployUtil.java deleted file mode 100644 index 550bddb..0000000 --- a/job/src/test/java/org/apache/kylin/job/DeployUtil.java +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.FileReader; -import java.io.IOException; -import java.io.InputStream; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.common.persistence.ResourceTool; -import org.apache.kylin.common.util.AbstractKylinTestCase; -import org.apache.kylin.common.util.CliCommandExecutor; -import org.apache.kylin.common.util.HiveClient; -import org.apache.kylin.common.util.Pair; -import org.apache.kylin.cube.CubeInstance; -import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.job.dataGen.FactTableGenerator; -import org.apache.kylin.job.hadoop.hive.SqlHiveDataTypeMapping; -import org.apache.kylin.metadata.MetadataManager; -import org.apache.kylin.metadata.model.ColumnDesc; -import org.apache.kylin.metadata.model.TableDesc; -import org.apache.maven.model.Model; -import org.apache.maven.model.io.xpp3.MavenXpp3Reader; -import org.apache.tools.ant.filters.StringInputStream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class DeployUtil { - @SuppressWarnings("unused") - private static final Logger logger = LoggerFactory.getLogger(DeployUtil.class); - - public static void initCliWorkDir() throws IOException { - execCliCommand("rm -rf " + getHadoopCliWorkingDir()); - execCliCommand("mkdir -p " + config().getKylinJobLogDir()); - } - - public static void deployMetadata() throws IOException { - // install metadata to hbase - ResourceTool.reset(config()); - ResourceTool.copy(KylinConfig.createInstanceFromUri(AbstractKylinTestCase.LOCALMETA_TEST_DATA), config()); - - // update cube desc signature. - for (CubeInstance cube : CubeManager.getInstance(config()).listAllCubes()) { - cube.getDescriptor().setSignature(cube.getDescriptor().calculateSignature()); - CubeManager.getInstance(config()).updateCube(cube); - } - } - - public static void overrideJobJarLocations() { - Pair<File, File> files = getJobJarFiles(); - File jobJar = files.getFirst(); - File coprocessorJar = files.getSecond(); - - config().overrideKylinJobJarPath(jobJar.getAbsolutePath()); - config().overrideCoprocessorLocalJar(coprocessorJar.getAbsolutePath()); - } - - public static void deployJobJars() throws IOException { - Pair<File, File> files = getJobJarFiles(); - File originalJobJar = files.getFirst(); - File originalCoprocessorJar = files.getSecond(); - - String jobJarPath = config().getKylinJobJarPath(); - if (StringUtils.isEmpty(jobJarPath)) { - throw new RuntimeException("deployJobJars cannot find job jar"); - } - - File targetJobJar = new File(jobJarPath); - File jobJarRenamedAsTarget = new File(originalJobJar.getParentFile(), targetJobJar.getName()); - if (originalJobJar.equals(jobJarRenamedAsTarget) == false) { - FileUtils.copyFile(originalJobJar, jobJarRenamedAsTarget); - } - - File targetCoprocessorJar = new File(config().getCoprocessorLocalJar()); - File coprocessorJarRenamedAsTarget = new File(originalCoprocessorJar.getParentFile(), targetCoprocessorJar.getName()); - if (originalCoprocessorJar.equals(coprocessorJarRenamedAsTarget) == false) { - FileUtils.copyFile(originalCoprocessorJar, coprocessorJarRenamedAsTarget); - } - - CliCommandExecutor cmdExec = config().getCliCommandExecutor(); - cmdExec.copyFile(jobJarRenamedAsTarget.getAbsolutePath(), targetJobJar.getParent()); - cmdExec.copyFile(coprocessorJarRenamedAsTarget.getAbsolutePath(), targetCoprocessorJar.getParent()); - } - - private static Pair<File, File> getJobJarFiles() { - String version; - try { - MavenXpp3Reader pomReader = new MavenXpp3Reader(); - Model model = pomReader.read(new FileReader("../pom.xml")); - version = model.getVersion(); - } catch (Exception e) { - throw new RuntimeException(e.getMessage(), e); - } - - File jobJar = new File("../job/target", "kylin-job-" + version + "-job.jar"); - File coprocessorJar = new File("../storage/target", "kylin-storage-" + version + "-coprocessor.jar"); - return new Pair<File, File>(jobJar, coprocessorJar); - } - - private static void execCliCommand(String cmd) throws IOException { - config().getCliCommandExecutor().execute(cmd); - } - - private static String getHadoopCliWorkingDir() { - return config().getCliWorkingDir(); - } - - private static KylinConfig config() { - return KylinConfig.getInstanceFromEnv(); - } - - // ============================================================================ - - static final String TABLE_CAL_DT = "edw.test_cal_dt"; - static final String TABLE_CATEGORY_GROUPINGS = "default.test_category_groupings"; - static final String TABLE_KYLIN_FACT = "default.test_kylin_fact"; - static final String TABLE_SELLER_TYPE_DIM = "edw.test_seller_type_dim"; - static final String TABLE_SITES = "edw.test_sites"; - - static final String[] TABLE_NAMES = new String[] { TABLE_CAL_DT, TABLE_CATEGORY_GROUPINGS, TABLE_KYLIN_FACT, TABLE_SELLER_TYPE_DIM, TABLE_SITES }; - - public static void prepareTestData(String joinType, String cubeName) throws Exception { - - String factTableName = TABLE_KYLIN_FACT.toUpperCase(); - String content = null; - - boolean buildCubeUsingProvidedData = Boolean.parseBoolean(System.getProperty("buildCubeUsingProvidedData")); - if (!buildCubeUsingProvidedData) { - System.out.println("build cube with random dataset"); - // data is generated according to cube descriptor and saved in resource store - if (joinType.equalsIgnoreCase("inner")) { - content = FactTableGenerator.generate(cubeName, "10000", "1", null, "inner"); - } else if (joinType.equalsIgnoreCase("left")) { - content = FactTableGenerator.generate(cubeName, "10000", "0.6", null, "left"); - } else { - throw new IllegalArgumentException("Unsupported join type : " + joinType); - } - - assert content != null; - overrideFactTableData(content, factTableName); - } else { - System.out.println("build cube with provided dataset"); - } - - duplicateFactTableData(factTableName, joinType); - deployHiveTables(); - } - - public static void overrideFactTableData(String factTableContent, String factTableName) throws IOException { - // Write to resource store - ResourceStore store = ResourceStore.getStore(config()); - - InputStream in = new StringInputStream(factTableContent); - String factTablePath = "/data/" + factTableName + ".csv"; - store.deleteResource(factTablePath); - store.putResource(factTablePath, in, System.currentTimeMillis()); - in.close(); - } - - public static void duplicateFactTableData(String factTableName, String joinType) throws IOException { - // duplicate a copy of this fact table, with a naming convention with fact.csv.inner or fact.csv.left - // so that later test cases can select different data files - ResourceStore store = ResourceStore.getStore(config()); - InputStream in = store.getResource("/data/" + factTableName + ".csv").inputStream; - String factTablePathWithJoinType = "/data/" + factTableName + ".csv." + joinType.toLowerCase(); - store.deleteResource(factTablePathWithJoinType); - store.putResource(factTablePathWithJoinType, in, System.currentTimeMillis()); - in.close(); - } - - private static void deployHiveTables() throws Exception { - - MetadataManager metaMgr = MetadataManager.getInstance(config()); - - // scp data files, use the data from hbase, instead of local files - File temp = File.createTempFile("temp", ".csv"); - temp.createNewFile(); - for (String tablename : TABLE_NAMES) { - tablename = tablename.toUpperCase(); - - File localBufferFile = new File(temp.getParent() + "/" + tablename + ".csv"); - localBufferFile.createNewFile(); - - InputStream hbaseDataStream = metaMgr.getStore().getResource("/data/" + tablename + ".csv").inputStream; - FileOutputStream localFileStream = new FileOutputStream(localBufferFile); - IOUtils.copy(hbaseDataStream, localFileStream); - - hbaseDataStream.close(); - localFileStream.close(); - - localBufferFile.deleteOnExit(); - } - String tableFileDir = temp.getParent(); - temp.delete(); - - HiveClient hiveClient = new HiveClient(); - - // create hive tables - hiveClient.executeHQL("CREATE DATABASE IF NOT EXISTS EDW"); - hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase()))); - hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CATEGORY_GROUPINGS.toUpperCase()))); - hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_KYLIN_FACT.toUpperCase()))); - hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SELLER_TYPE_DIM.toUpperCase()))); - hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SITES.toUpperCase()))); - - // load data to hive tables - // LOAD DATA LOCAL INPATH 'filepath' [OVERWRITE] INTO TABLE tablename - hiveClient.executeHQL(generateLoadDataHql(TABLE_CAL_DT, tableFileDir)); - hiveClient.executeHQL(generateLoadDataHql(TABLE_CATEGORY_GROUPINGS, tableFileDir)); - hiveClient.executeHQL(generateLoadDataHql(TABLE_KYLIN_FACT, tableFileDir)); - hiveClient.executeHQL(generateLoadDataHql(TABLE_SELLER_TYPE_DIM, tableFileDir)); - hiveClient.executeHQL(generateLoadDataHql(TABLE_SITES, tableFileDir)); - } - - private static String generateLoadDataHql(String tableName, String tableFileDir) { - return "LOAD DATA LOCAL INPATH '" + tableFileDir + "/" + tableName.toUpperCase() + ".csv' OVERWRITE INTO TABLE " + tableName.toUpperCase(); - } - - private static String[] generateCreateTableHql(TableDesc tableDesc) { - - String dropsql = "DROP TABLE IF EXISTS " + tableDesc.getIdentity(); - StringBuilder ddl = new StringBuilder(); - - ddl.append("CREATE TABLE " + tableDesc.getIdentity() + "\n"); - ddl.append("(" + "\n"); - - for (int i = 0; i < tableDesc.getColumns().length; i++) { - ColumnDesc col = tableDesc.getColumns()[i]; - if (i > 0) { - ddl.append(","); - } - ddl.append(col.getName() + " " + SqlHiveDataTypeMapping.getHiveDataType((col.getDatatype())) + "\n"); - } - - ddl.append(")" + "\n"); - ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n"); - ddl.append("STORED AS TEXTFILE"); - - return new String[] { dropsql, ddl.toString() }; - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/ErrorTestExecutable.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/ErrorTestExecutable.java b/job/src/test/java/org/apache/kylin/job/ErrorTestExecutable.java deleted file mode 100644 index 56df524..0000000 --- a/job/src/test/java/org/apache/kylin/job/ErrorTestExecutable.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job; - -import org.apache.kylin.job.exception.ExecuteException; -import org.apache.kylin.job.execution.ExecutableContext; -import org.apache.kylin.job.execution.ExecuteResult; - -/** - * Created by qianzhou on 12/22/14. - */ -public class ErrorTestExecutable extends BaseTestExecutable { - - public ErrorTestExecutable() { - super(); - } - - @Override - protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - } - throw new RuntimeException("test error"); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java b/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java deleted file mode 100644 index e784a41..0000000 --- a/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job; - -import java.io.File; -import java.io.IOException; - -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HConnection; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.HBaseConnection; -import org.apache.kylin.common.util.AbstractKylinTestCase; -import org.apache.kylin.common.util.CliCommandExecutor; -import org.apache.kylin.common.util.HBaseMiniclusterHelper; -import org.apache.kylin.common.util.SSHClient; -import org.apache.kylin.job.constant.BatchConstants; - -public class ExportHBaseData { - - KylinConfig kylinConfig; - HTableDescriptor[] allTables; - Configuration config; - HBaseAdmin hbase; - CliCommandExecutor cli; - String exportHdfsFolder; - String exportLocalFolderParent; - String exportLocalFolder; - String backupArchive; - String tableNameBase; - long currentTIME; - - public ExportHBaseData() { - try { - setup(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private void setup() throws IOException { - - KylinConfig.destoryInstance(); - System.setProperty(KylinConfig.KYLIN_CONF, AbstractKylinTestCase.SANDBOX_TEST_DATA); - - kylinConfig = KylinConfig.getInstanceFromEnv(); - cli = kylinConfig.getCliCommandExecutor(); - - currentTIME = System.currentTimeMillis(); - exportHdfsFolder = kylinConfig.getHdfsWorkingDirectory() + "hbase-export/" + currentTIME + "/"; - exportLocalFolderParent = BatchConstants.CFG_KYLIN_LOCAL_TEMP_DIR + "hbase-export/"; - exportLocalFolder = exportLocalFolderParent + currentTIME + "/"; - backupArchive = exportLocalFolderParent + "hbase-export-at-" + currentTIME + ".tar.gz"; - - String metadataUrl = kylinConfig.getMetadataUrl(); - // split TABLE@HBASE_URL - int cut = metadataUrl.indexOf('@'); - tableNameBase = metadataUrl.substring(0, cut); - String hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1); - - HConnection conn = HBaseConnection.get(hbaseUrl); - try { - hbase = new HBaseAdmin(conn); - config = hbase.getConfiguration(); - allTables = hbase.listTables(); - } catch (IOException e) { - e.printStackTrace(); - throw e; - } - } - - public void tearDown() { - - // cleanup hdfs - try { - if (cli != null && exportHdfsFolder != null) { - cli.execute("hadoop fs -rm -r " + exportHdfsFolder); - } - } catch (IOException e) { - e.printStackTrace(); - } - // cleanup sandbox disk - try { - if (cli != null && exportLocalFolder != null) { - cli.execute("rm -r " + exportLocalFolder); - } - } catch (IOException e) { - e.printStackTrace(); - } - - // delete archive file on sandbox - try { - if (cli != null && backupArchive != null) { - cli.execute("rm " + backupArchive); - } - } catch (IOException e) { - e.printStackTrace(); - } - } - - public void exportTables() throws IOException { - cli.execute("mkdir -p " + exportLocalFolderParent); - - for (HTableDescriptor table : allTables) { - String tName = table.getNameAsString(); - if (!tName.equals(tableNameBase) && !tName.startsWith(HBaseMiniclusterHelper.SHARED_STORAGE_PREFIX)) - continue; - - cli.execute("hbase org.apache.hadoop.hbase.mapreduce.Export " + tName + " " + exportHdfsFolder + tName); - } - - cli.execute("hadoop fs -copyToLocal " + exportHdfsFolder + " " + exportLocalFolderParent); - cli.execute("tar -zcvf " + backupArchive + " --directory=" + exportLocalFolderParent + " " + currentTIME); - - downloadToLocal(); - } - - public void downloadToLocal() throws IOException { - String localArchive = "../examples/test_case_data/minicluster/hbase-export.tar.gz"; - - if (kylinConfig.getRunAsRemoteCommand()) { - SSHClient ssh = new SSHClient(kylinConfig.getRemoteHadoopCliHostname(), kylinConfig.getRemoteHadoopCliUsername(), kylinConfig.getRemoteHadoopCliPassword()); - try { - ssh.scpFileToLocal(backupArchive, localArchive); - } catch (Exception e) { - e.printStackTrace(); - } - } else { - FileUtils.copyFile(new File(backupArchive), new File(localArchive)); - } - } - - public static void main(String[] args) { - ExportHBaseData export = new ExportHBaseData(); - try { - export.exportTables(); - } catch (IOException e) { - e.printStackTrace(); - } finally { - export.tearDown(); - } - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/FailedTestExecutable.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/FailedTestExecutable.java b/job/src/test/java/org/apache/kylin/job/FailedTestExecutable.java deleted file mode 100644 index a224214..0000000 --- a/job/src/test/java/org/apache/kylin/job/FailedTestExecutable.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job; - -import org.apache.kylin.job.exception.ExecuteException; -import org.apache.kylin.job.execution.ExecutableContext; -import org.apache.kylin.job.execution.ExecuteResult; - -/** - * Created by qianzhou on 12/22/14. - */ -public class FailedTestExecutable extends BaseTestExecutable { - - public FailedTestExecutable() { - super(); - } - - @Override - protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - } - return new ExecuteResult(ExecuteResult.State.FAILED, "failed"); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java b/job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java deleted file mode 100644 index 53d9ceb..0000000 --- a/job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job; - -import org.apache.kylin.job.exception.ExecuteException; -import org.apache.kylin.job.execution.ExecutableContext; -import org.apache.kylin.job.execution.ExecuteResult; - -/** - * Created by qianzhou on 12/31/14. - */ -public class SelfStopExecutable extends BaseTestExecutable { - - public SelfStopExecutable() { - super(); - } - - @Override - protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { - try { - Thread.sleep(5000); - } catch (InterruptedException e) { - } - if (isDiscarded()) { - return new ExecuteResult(ExecuteResult.State.STOPPED, "stopped"); - } else { - return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed"); - } - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/SucceedTestExecutable.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/SucceedTestExecutable.java b/job/src/test/java/org/apache/kylin/job/SucceedTestExecutable.java deleted file mode 100644 index 57a62b9..0000000 --- a/job/src/test/java/org/apache/kylin/job/SucceedTestExecutable.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job; - -import org.apache.kylin.job.exception.ExecuteException; -import org.apache.kylin.job.execution.ExecutableContext; -import org.apache.kylin.job.execution.ExecuteResult; - -/** - * Created by qianzhou on 12/22/14. - */ -public class SucceedTestExecutable extends BaseTestExecutable { - - public SucceedTestExecutable() { - super(); - } - - @Override - protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - } - return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed"); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/dataGen/ColumnConfig.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/dataGen/ColumnConfig.java b/job/src/test/java/org/apache/kylin/job/dataGen/ColumnConfig.java deleted file mode 100644 index 6f12d30..0000000 --- a/job/src/test/java/org/apache/kylin/job/dataGen/ColumnConfig.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.dataGen; - -import java.util.ArrayList; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * Created by honma on 5/29/14. - */ -@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE) -public class ColumnConfig { - @JsonProperty("columnName") - private String columnName; - @JsonProperty("valueSet") - private ArrayList<String> valueSet; - @JsonProperty("exclusive") - private boolean exclusive; - @JsonProperty("asRange") - private boolean asRange; - - public boolean isAsRange() { - return asRange; - } - - public void setAsRange(boolean asRange) { - this.asRange = asRange; - } - - public boolean isExclusive() { - return exclusive; - } - - public void setExclusive(boolean exclusive) { - this.exclusive = exclusive; - } - - public String getColumnName() { - return columnName; - } - - public void setColumnName(String columnName) { - this.columnName = columnName; - } - - public ArrayList<String> getValueSet() { - return valueSet; - } - - public void setValueSet(ArrayList<String> valueSet) { - this.valueSet = valueSet; - } - -}
