Repository: kylin Updated Branches: refs/heads/master 49a0a41f0 -> bb1cbda97
code refactor: make all tool CLI subclass of AbstractApplication Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bb1cbda9 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bb1cbda9 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bb1cbda9 Branch: refs/heads/master Commit: bb1cbda97ba9278e3ee77f1df593cc533b89dc75 Parents: 49a0a41 Author: Hongbin Ma <[email protected]> Authored: Thu Jul 14 14:49:03 2016 +0800 Committer: Hongbin Ma <[email protected]> Committed: Thu Jul 14 14:49:03 2016 +0800 ---------------------------------------------------------------------- .../kylin/job/execution/AbstractExecutable.java | 2 +- .../engine/mr/common/AbstractHadoopJob.java | 1 - .../mr/steps/RowKeyDistributionCheckerJob.java | 6 +- .../kylin/provision/BuildCubeWithEngine.java | 6 +- .../kylin/provision/BuildCubeWithStream.java | 6 +- .../apache/kylin/rest/service/AdminService.java | 3 +- .../storage/hbase/util/CleanHtableCLI.java | 35 +++++------ .../kylin/storage/hbase/util/HBaseClean.java | 53 +++++++--------- .../hbase/util/HtableAlterMetadataCLI.java | 53 ++++++++-------- .../storage/hbase/util/OrphanHBaseCleanJob.java | 66 +++++++++----------- .../storage/hbase/util/StorageCleanupJob.java | 65 ++++++++----------- 11 files changed, 131 insertions(+), 165 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/bb1cbda9/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java index ab70007..4dedad1 100644 --- a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java +++ b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java @@ -95,7 +95,7 @@ public abstract class AbstractExecutable implements Executable, Idempotent { @Override public final ExecuteResult execute(ExecutableContext executableContext) throws ExecuteException { - logger.info("Executing >>>>>>>>>>>>> " + this.getName() + " <<<<<<<<<<<<<"); + logger.info("Executing AbstractExecutable (" + this.getName() + ")"); Preconditions.checkArgument(executableContext instanceof DefaultContext); ExecuteResult result = null; http://git-wip-us.apache.org/repos/asf/kylin/blob/bb1cbda9/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java index bafe65b..b483dc1 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java @@ -77,7 +77,6 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { protected static final Option OPTION_JOB_NAME = OptionBuilder.withArgName(BatchConstants.ARG_JOB_NAME).hasArg().isRequired(true).withDescription("Job name. For example, Kylin_Cuboid_Builder-clsfd_v2_Step_22-D)").create(BatchConstants.ARG_JOB_NAME); protected static final Option OPTION_CUBE_NAME = OptionBuilder.withArgName(BatchConstants.ARG_CUBE_NAME).hasArg().isRequired(true).withDescription("Cube name. For exmaple, flat_item_cube").create(BatchConstants.ARG_CUBE_NAME); protected static final Option OPTION_CUBING_JOB_ID = OptionBuilder.withArgName(BatchConstants.ARG_CUBING_JOB_ID).hasArg().isRequired(false).withDescription("ID of cubing job executable").create(BatchConstants.ARG_CUBING_JOB_ID); - protected static final Option OPTION_II_NAME = OptionBuilder.withArgName(BatchConstants.ARG_II_NAME).hasArg().isRequired(true).withDescription("II name. For exmaple, some_ii").create(BatchConstants.ARG_II_NAME); protected static final Option OPTION_SEGMENT_NAME = OptionBuilder.withArgName(BatchConstants.ARG_SEGMENT_NAME).hasArg().isRequired(true).withDescription("Cube segment name").create(BatchConstants.ARG_SEGMENT_NAME); protected static final Option OPTION_INPUT_PATH = OptionBuilder.withArgName(BatchConstants.ARG_INPUT).hasArg().isRequired(true).withDescription("Input path").create(BatchConstants.ARG_INPUT); protected static final Option OPTION_INPUT_FORMAT = OptionBuilder.withArgName(BatchConstants.ARG_INPUT_FORMAT).hasArg().isRequired(false).withDescription("Input format").create(BatchConstants.ARG_INPUT_FORMAT); http://git-wip-us.apache.org/repos/asf/kylin/blob/bb1cbda9/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerJob.java index c5a0ff2..fc8d3e7 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerJob.java @@ -39,7 +39,7 @@ import org.apache.kylin.engine.mr.common.AbstractHadoopJob; public class RowKeyDistributionCheckerJob extends AbstractHadoopJob { @SuppressWarnings("static-access") - protected static final Option rowKeyStatsFilePath = OptionBuilder.withArgName("path").hasArg().isRequired(true).withDescription("rowKeyStatsFilePath").create("rowKeyStatsFilePath"); + protected static final Option ROW_KEY_STATS_FILE_PATH = OptionBuilder.withArgName("path").hasArg().isRequired(true).withDescription("rowKeyStatsFilePath").create("rowKeyStatsFilePath"); @Override public int run(String[] args) throws Exception { @@ -49,11 +49,11 @@ public class RowKeyDistributionCheckerJob extends AbstractHadoopJob { options.addOption(OPTION_INPUT_PATH); options.addOption(OPTION_OUTPUT_PATH); options.addOption(OPTION_JOB_NAME); - options.addOption(rowKeyStatsFilePath); + options.addOption(ROW_KEY_STATS_FILE_PATH); parseOptions(options, args); - String statsFilePath = getOptionValue(rowKeyStatsFilePath); + String statsFilePath = getOptionValue(ROW_KEY_STATS_FILE_PATH); // start job String jobName = getOptionValue(OPTION_JOB_NAME); http://git-wip-us.apache.org/repos/asf/kylin/blob/bb1cbda9/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java index 9ce2bf2..3d60a3c 100644 --- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java +++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java @@ -37,7 +37,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.util.ToolRunner; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.ClassUtil; import org.apache.kylin.common.util.HBaseMetadataTestCase; @@ -414,8 +413,9 @@ public class BuildCubeWithEngine { private int cleanupOldStorage() throws Exception { String[] args = { "--delete", "true" }; - int exitCode = ToolRunner.run(new StorageCleanupJob(), args); - return exitCode; + StorageCleanupJob cli = new StorageCleanupJob(); + cli.execute(args); + return 0; } private void checkHFilesInHBase(CubeSegment segment) throws IOException { http://git-wip-us.apache.org/repos/asf/kylin/blob/bb1cbda9/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java index 1655a17..d41dc34 100644 --- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java +++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.util.UUID; import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.util.ToolRunner; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.ClassUtil; import org.apache.kylin.common.util.DateFormat; @@ -111,8 +110,9 @@ public class BuildCubeWithStream { private static int cleanupOldStorage() throws Exception { String[] args = { "--delete", "true" }; - int exitCode = ToolRunner.run(new StorageCleanupJob(), args); - return exitCode; + StorageCleanupJob cli = new StorageCleanupJob(); + cli.execute(args); + return 0; } public void build() throws Exception { http://git-wip-us.apache.org/repos/asf/kylin/blob/bb1cbda9/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java index 6c85898..b92c305 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java @@ -25,7 +25,6 @@ import java.util.Properties; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.hadoop.util.ToolRunner; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.rest.constant.Constant; import org.apache.kylin.rest.exception.InternalErrorException; @@ -101,7 +100,7 @@ public class AdminService extends BasicService { StorageCleanupJob job = new StorageCleanupJob(); String[] args = new String[] { "-delete", "true" }; try { - ToolRunner.run(job, args); + job.execute(args); } catch (Exception e) { throw new InternalErrorException(e.getMessage(), e); } http://git-wip-us.apache.org/repos/asf/kylin/blob/bb1cbda9/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java index 9e30a06..a150607 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java @@ -24,8 +24,8 @@ import org.apache.commons.cli.Options; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.util.ToolRunner; -import org.apache.kylin.engine.mr.common.AbstractHadoopJob; +import org.apache.kylin.common.util.AbstractApplication; +import org.apache.kylin.common.util.OptionsHelper; import org.apache.kylin.metadata.realization.IRealizationConstants; import org.apache.kylin.storage.hbase.HBaseConnection; import org.slf4j.Logger; @@ -33,24 +33,10 @@ import org.slf4j.LoggerFactory; /** */ -public class CleanHtableCLI extends AbstractHadoopJob { +public class CleanHtableCLI extends AbstractApplication { protected static final Logger logger = LoggerFactory.getLogger(CleanHtableCLI.class); - @Override - public int run(String[] args) throws Exception { - Options options = new Options(); - try { - - clean(); - - return 0; - } catch (Exception e) { - printUsage(options); - throw e; - } - } - private void clean() throws IOException { Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); HBaseAdmin hbaseAdmin = new HBaseAdmin(conf); @@ -71,7 +57,18 @@ public class CleanHtableCLI extends AbstractHadoopJob { } public static void main(String[] args) throws Exception { - int exitCode = ToolRunner.run(new CleanHtableCLI(), args); - System.exit(exitCode); + CleanHtableCLI cli = new CleanHtableCLI(); + cli.execute(args); + } + + @Override + protected Options getOptions() { + Options options = new Options(); + return options; + } + + @Override + protected void execute(OptionsHelper optionsHelper) throws Exception { + clean(); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/bb1cbda9/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java index b6958d6..6749d6c 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java @@ -19,7 +19,6 @@ package org.apache.kylin.storage.hbase.util; import java.io.IOException; -import java.util.Arrays; import java.util.List; import org.apache.commons.cli.Option; @@ -28,8 +27,8 @@ import org.apache.commons.cli.Options; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.util.ToolRunner; -import org.apache.kylin.engine.mr.common.AbstractHadoopJob; +import org.apache.kylin.common.util.AbstractApplication; +import org.apache.kylin.common.util.OptionsHelper; import org.apache.kylin.metadata.realization.IRealizationConstants; import org.apache.kylin.storage.hbase.HBaseConnection; import org.slf4j.Logger; @@ -40,7 +39,7 @@ import com.google.common.collect.Lists; /** * clean hbase tables by tag */ -public class HBaseClean extends AbstractHadoopJob { +public class HBaseClean extends AbstractApplication { @SuppressWarnings("static-access") private static final Option OPTION_DELETE = OptionBuilder.withArgName("delete").hasArg().isRequired(true).withDescription("actually delete or not").create("delete"); @@ -49,33 +48,10 @@ public class HBaseClean extends AbstractHadoopJob { private static final Option OPTION_TAG = OptionBuilder.withArgName("tag").hasArg().isRequired(true).withDescription("the tag of HTable").create("tag"); protected static final Logger logger = LoggerFactory.getLogger(HBaseClean.class); + boolean delete = false; String tag = null; - @Override - public int run(String[] args) throws Exception { - Options options = new Options(); - - logger.info("jobs args: " + Arrays.toString(args)); - try { - options.addOption(OPTION_DELETE); - options.addOption(OPTION_TAG); - parseOptions(options, args); - - logger.info("options: '" + getOptionsAsString() + "'"); - - tag = getOptionValue(OPTION_TAG); - delete = Boolean.parseBoolean(getOptionValue(OPTION_DELETE)); - - cleanUp(); - - return 0; - } catch (Exception e) { - e.printStackTrace(System.err); - throw e; - } - } - private void cleanUp() { try { // get all kylin hbase tables @@ -121,7 +97,24 @@ public class HBaseClean extends AbstractHadoopJob { } public static void main(String[] args) throws Exception { - int exitCode = ToolRunner.run(new HBaseClean(), args); - System.exit(exitCode); + HBaseClean cli = new HBaseClean(); + cli.execute(args); + } + + @Override + protected Options getOptions() { + Options options = new Options(); + options.addOption(OPTION_DELETE); + options.addOption(OPTION_TAG); + return options; + + } + + @Override + protected void execute(OptionsHelper optionsHelper) throws Exception { + tag = optionsHelper.getOptionValue(OPTION_TAG); + delete = Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_DELETE)); + + cleanUp(); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/bb1cbda9/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java index 4db183b..ca1a060 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java @@ -27,8 +27,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.util.ToolRunner; -import org.apache.kylin.engine.mr.common.AbstractHadoopJob; +import org.apache.kylin.common.util.AbstractApplication; +import org.apache.kylin.common.util.OptionsHelper; +import org.apache.kylin.engine.mr.common.BatchConstants; import org.apache.kylin.storage.hbase.HBaseConnection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,10 +37,11 @@ import org.slf4j.LoggerFactory; /** */ @SuppressWarnings("static-access") -public class HtableAlterMetadataCLI extends AbstractHadoopJob { +public class HtableAlterMetadataCLI extends AbstractApplication { private static final Option OPTION_METADATA_KEY = OptionBuilder.withArgName("key").hasArg().isRequired(true).withDescription("The metadata key").create("key"); private static final Option OPTION_METADATA_VALUE = OptionBuilder.withArgName("value").hasArg().isRequired(true).withDescription("The metadata value").create("value"); + protected static final Option OPTION_HTABLE_NAME = OptionBuilder.withArgName(BatchConstants.ARG_HTABLE_NAME).hasArg().isRequired(true).withDescription("HTable name").create(BatchConstants.ARG_HTABLE_NAME); protected static final Logger logger = LoggerFactory.getLogger(HtableAlterMetadataCLI.class); @@ -47,28 +49,6 @@ public class HtableAlterMetadataCLI extends AbstractHadoopJob { String metadataKey; String metadataValue; - @Override - public int run(String[] args) throws Exception { - Options options = new Options(); - try { - options.addOption(OPTION_HTABLE_NAME); - options.addOption(OPTION_METADATA_KEY); - options.addOption(OPTION_METADATA_VALUE); - - parseOptions(options, args); - tableName = getOptionValue(OPTION_HTABLE_NAME); - metadataKey = getOptionValue(OPTION_METADATA_KEY); - metadataValue = getOptionValue(OPTION_METADATA_VALUE); - - alter(); - - return 0; - } catch (Exception e) { - printUsage(options); - throw e; - } - } - private void alter() throws IOException { Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); HBaseAdmin hbaseAdmin = new HBaseAdmin(conf); @@ -82,7 +62,26 @@ public class HtableAlterMetadataCLI extends AbstractHadoopJob { } public static void main(String[] args) throws Exception { - int exitCode = ToolRunner.run(new HtableAlterMetadataCLI(), args); - System.exit(exitCode); + HtableAlterMetadataCLI cli = new HtableAlterMetadataCLI(); + cli.execute(args); + } + + @Override + protected Options getOptions() { + Options options = new Options(); + options.addOption(OPTION_HTABLE_NAME); + options.addOption(OPTION_METADATA_KEY); + options.addOption(OPTION_METADATA_VALUE); + return options; + } + + @Override + protected void execute(OptionsHelper optionsHelper) throws Exception { + tableName = optionsHelper.getOptionValue(OPTION_HTABLE_NAME); + metadataKey = optionsHelper.getOptionValue(OPTION_METADATA_KEY); + metadataValue = optionsHelper.getOptionValue(OPTION_METADATA_VALUE); + + alter(); + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/bb1cbda9/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java index f0618c9..8ff5b0f 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java @@ -20,7 +20,6 @@ package org.apache.kylin.storage.hbase.util; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Set; import java.util.TreeSet; @@ -32,15 +31,15 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.util.ToolRunner; -import org.apache.kylin.engine.mr.common.AbstractHadoopJob; +import org.apache.kylin.common.util.AbstractApplication; +import org.apache.kylin.common.util.OptionsHelper; import org.apache.kylin.metadata.realization.IRealizationConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** */ -public class OrphanHBaseCleanJob extends AbstractHadoopJob { +public class OrphanHBaseCleanJob extends AbstractApplication { @SuppressWarnings("static-access") private static final Option OPTION_DELETE = OptionBuilder.withArgName("delete").hasArg().isRequired(false).withDescription("Delete the unused storage").create("delete"); @@ -52,37 +51,6 @@ public class OrphanHBaseCleanJob extends AbstractHadoopJob { boolean delete = false; Set<String> metastoreWhitelistSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - @Override - public int run(String[] args) throws Exception { - Options options = new Options(); - - logger.info("jobs args: " + Arrays.toString(args)); - try { - options.addOption(OPTION_DELETE); - options.addOption(OPTION_WHITELIST); - parseOptions(options, args); - - logger.info("options: '" + getOptionsAsString() + "'"); - logger.info("delete option value: '" + getOptionValue(OPTION_DELETE) + "'"); - delete = Boolean.parseBoolean(getOptionValue(OPTION_DELETE)); - String[] metastoreWhitelist = getOptionValue(OPTION_WHITELIST).split(","); - - for (String ms : metastoreWhitelist) { - logger.info("metadata store in white list: " + ms); - metastoreWhitelistSet.add(ms); - } - - Configuration conf = HBaseConfiguration.create(getConf()); - - cleanUnusedHBaseTables(conf); - - return 0; - } catch (Exception e) { - e.printStackTrace(System.err); - throw e; - } - } - private void cleanUnusedHBaseTables(Configuration conf) throws IOException { // get all kylin hbase tables @@ -128,7 +96,31 @@ public class OrphanHBaseCleanJob extends AbstractHadoopJob { } public static void main(String[] args) throws Exception { - int exitCode = ToolRunner.run(new OrphanHBaseCleanJob(), args); - System.exit(exitCode); + OrphanHBaseCleanJob job = new OrphanHBaseCleanJob(); + job.execute(args); + } + + @Override + protected Options getOptions() { + Options options = new Options(); + options.addOption(OPTION_DELETE); + options.addOption(OPTION_WHITELIST); + return options; + } + + @Override + protected void execute(OptionsHelper optionsHelper) throws Exception { + logger.info("options: '" + optionsHelper.getOptionsAsString() + "'"); + logger.info("delete option value: '" + optionsHelper.getOptionValue(OPTION_DELETE) + "'"); + delete = Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_DELETE)); + String[] metastoreWhitelist = optionsHelper.getOptionValue(OPTION_WHITELIST).split(","); + + for (String ms : metastoreWhitelist) { + logger.info("metadata store in white list: " + ms); + metastoreWhitelistSet.add(ms); + } + + Configuration conf = HBaseConfiguration.create(); + cleanUnusedHBaseTables(conf); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/bb1cbda9/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java index 3f4a6d5..214375a 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java @@ -22,7 +22,6 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.StringReader; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; @@ -42,14 +41,14 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.util.ToolRunner; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.util.AbstractApplication; import org.apache.kylin.common.util.CliCommandExecutor; +import org.apache.kylin.common.util.OptionsHelper; import org.apache.kylin.common.util.Pair; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.CubeSegment; -import org.apache.kylin.engine.mr.common.AbstractHadoopJob; import org.apache.kylin.job.JobInstance; import org.apache.kylin.job.engine.JobEngineConfig; import org.apache.kylin.job.execution.ExecutableState; @@ -58,50 +57,17 @@ import org.apache.kylin.metadata.realization.IRealizationConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class StorageCleanupJob extends AbstractHadoopJob { +public class StorageCleanupJob extends AbstractApplication { @SuppressWarnings("static-access") private static final Option OPTION_DELETE = OptionBuilder.withArgName("delete").hasArg().isRequired(false).withDescription("Delete the unused storage").create("delete"); protected static final Logger logger = LoggerFactory.getLogger(StorageCleanupJob.class); - public static final int TIME_THRESHOLD_DELETE_HTABLE = 10; // Unit minute boolean delete = false; - protected static ExecutableManager executableManager = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv()); - /* - * (non-Javadoc) - * - * @see org.apache.hadoop.util.Tool#run(java.lang.String[]) - */ - @Override - public int run(String[] args) throws Exception { - Options options = new Options(); - - logger.info("jobs args: " + Arrays.toString(args)); - try { - options.addOption(OPTION_DELETE); - parseOptions(options, args); - - logger.info("options: '" + getOptionsAsString() + "'"); - logger.info("delete option value: '" + getOptionValue(OPTION_DELETE) + "'"); - delete = Boolean.parseBoolean(getOptionValue(OPTION_DELETE)); - - Configuration conf = HBaseConfiguration.create(getConf()); - - cleanUnusedIntermediateHiveTable(conf); - cleanUnusedHdfsFiles(conf); - cleanUnusedHBaseTables(conf); - - return 0; - } catch (Exception e) { - printUsage(options); - throw e; - } - } - private void cleanUnusedHBaseTables(Configuration conf) throws IOException { CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); long TIME_THREADSHOLD = KylinConfig.getInstanceFromEnv().getStorageCleanupTimeThreshold(); @@ -162,6 +128,27 @@ public class StorageCleanupJob extends AbstractHadoopJob { hbaseAdmin.close(); } + @Override + protected Options getOptions() { + Options options = new Options(); + options.addOption(OPTION_DELETE); + return options; + } + + @Override + protected void execute(OptionsHelper optionsHelper) throws Exception { + logger.info("options: '" + optionsHelper.getOptionsAsString() + "'"); + logger.info("delete option value: '" + optionsHelper.getOptionValue(OPTION_DELETE) + "'"); + delete = Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_DELETE)); + + Configuration conf = HBaseConfiguration.create(); + + cleanUnusedIntermediateHiveTable(conf); + cleanUnusedHdfsFiles(conf); + cleanUnusedHBaseTables(conf); + + } + class DeleteHTableRunnable implements Callable { HBaseAdmin hbaseAdmin; String htableName; @@ -322,7 +309,7 @@ public class StorageCleanupJob extends AbstractHadoopJob { } public static void main(String[] args) throws Exception { - int exitCode = ToolRunner.run(new StorageCleanupJob(), args); - System.exit(exitCode); + StorageCleanupJob cli = new StorageCleanupJob(); + cli.execute(args); } }
