Repository: kylin Updated Branches: refs/heads/master e4624779b -> ec2603ea4
KYLIN-2857 make -conf, -cubename optional Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ec2603ea Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ec2603ea Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ec2603ea Branch: refs/heads/master Commit: ec2603ea46fbdb0e778d0257fa23e56fbe5aa762 Parents: e462477 Author: Li Yang <liy...@apache.org> Authored: Mon Sep 18 06:37:57 2017 +0800 Committer: Li Yang <liy...@apache.org> Committed: Mon Sep 18 06:37:57 2017 +0800 ---------------------------------------------------------------------- .../engine/mr/common/AbstractHadoopJob.java | 2 - .../engine/mr/common/MapReduceExecutable.java | 43 ++++++++++++++------ 2 files changed, 30 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/ec2603ea/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java index 203b2df..babf69b 100755 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java @@ -112,8 +112,6 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { protected static final Option OPTION_STATISTICS_SAMPLING_PERCENT = OptionBuilder .withArgName(BatchConstants.ARG_STATS_SAMPLING_PERCENT).hasArg().isRequired(false) .withDescription("Statistics sampling percentage").create(BatchConstants.ARG_STATS_SAMPLING_PERCENT); - protected static final Option OPTION_JOB_CONF = OptionBuilder.withArgName(BatchConstants.ARG_CONF).hasArg() - .isRequired(true).withDescription("MapReduce job config file").create(BatchConstants.ARG_CONF); private static final String MAP_REDUCE_CLASSPATH = "mapreduce.application.classpath"; http://git-wip-us.apache.org/repos/asf/kylin/blob/ec2603ea/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java index 94874dc..af02364 100755 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java @@ -28,6 +28,8 @@ import java.util.Map; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.lang.StringUtils; @@ -244,26 +246,41 @@ public class MapReduceExecutable extends AbstractExecutable { setParam(KEY_COUNTER_SAVEAS, value); } + @SuppressWarnings("static-access") + private static final Option OPTION_JOB_CONF = OptionBuilder.withArgName(BatchConstants.ARG_CONF).hasArg() + .isRequired(false).create(BatchConstants.ARG_CONF); + + @SuppressWarnings("static-access") + private static final Option OPTION_CUBE_NAME = OptionBuilder.withArgName(BatchConstants.ARG_CUBE_NAME).hasArg() + .isRequired(false).create(BatchConstants.ARG_CUBE_NAME); + private String[] overwriteJobConf(Configuration conf, KylinConfig config, String[] jobParams) throws ParseException { Options options = new Options(); - options.addOption(AbstractHadoopJob.OPTION_JOB_CONF); - options.addOption(AbstractHadoopJob.OPTION_CUBE_NAME); + options.addOption(OPTION_JOB_CONF); + options.addOption(OPTION_CUBE_NAME); CustomParser parser = new CustomParser(); CommandLine commandLine = parser.parse(options, jobParams); - String fileName = commandLine.getOptionValue(BatchConstants.ARG_CONF); + + String confFile = commandLine.getOptionValue(BatchConstants.ARG_CONF); String cubeName = commandLine.getOptionValue(BatchConstants.ARG_CUBE_NAME); - Preconditions.checkArgument(cubeName != null && fileName != null, "Can't get job config"); - conf.addResource(new Path(fileName)); - for (Map.Entry<String, String> entry : CubeManager.getInstance(config).getCube(cubeName).getConfig() - .getMRConfigOverride().entrySet()) { - conf.set(entry.getKey(), entry.getValue()); + List<String> remainingArgs = Lists.newArrayList(); + + if (StringUtils.isNotBlank(confFile)) { + conf.addResource(new Path(confFile)); + } + + if (StringUtils.isNotBlank(cubeName)) { + for (Map.Entry<String, String> entry : CubeManager.getInstance(config).getCube(cubeName).getConfig() + .getMRConfigOverride().entrySet()) { + conf.set(entry.getKey(), entry.getValue()); + } + remainingArgs.add("-" + BatchConstants.ARG_CUBE_NAME); + remainingArgs.add(cubeName); } - List<String> remainingArgs = parser.getRemainingArgs(); - remainingArgs.add("-" + BatchConstants.ARG_CUBE_NAME); - remainingArgs.add(cubeName); - String[] result = new String[remainingArgs.size()]; - return remainingArgs.toArray(result); + + remainingArgs.addAll(parser.getRemainingArgs()); + return (String[]) remainingArgs.toArray(new String[remainingArgs.size()]); } private static class CustomParser extends GnuParser {
