KYLIN-1507 find hive dependency jar on some platform like CDH
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dba135f4 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dba135f4 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dba135f4 Branch: refs/heads/master Commit: dba135f4c8d834fce53f0d3490a77af6e204e24e Parents: 30cccbd Author: Yang Li <[email protected]> Authored: Sun Mar 20 20:13:08 2016 +0800 Committer: Yang Li <[email protected]> Committed: Sun Mar 20 20:13:08 2016 +0800 ---------------------------------------------------------------------- .../engine/mr/common/AbstractHadoopJob.java | 68 ++++++++++---------- 1 file changed, 35 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/dba135f4/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java index fe60ca8..04b591f 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java @@ -78,7 +78,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { protected static final Option OPTION_CUBE_NAME = OptionBuilder.withArgName(BatchConstants.ARG_CUBE_NAME).hasArg().isRequired(true).withDescription("Cube name. For exmaple, flat_item_cube").create(BatchConstants.ARG_CUBE_NAME); protected static final Option OPTION_CUBING_JOB_ID = OptionBuilder.withArgName(BatchConstants.ARG_CUBING_JOB_ID).hasArg().isRequired(false).withDescription("ID of cubing job executable").create(BatchConstants.ARG_CUBING_JOB_ID); protected static final Option OPTION_II_NAME = OptionBuilder.withArgName(BatchConstants.ARG_II_NAME).hasArg().isRequired(true).withDescription("II name. For exmaple, some_ii").create(BatchConstants.ARG_II_NAME); - protected static final Option OPTION_SEGMENT_NAME = OptionBuilder.withArgName( BatchConstants.ARG_SEGMENT_NAME).hasArg().isRequired(true).withDescription("Cube segment name").create( BatchConstants.ARG_SEGMENT_NAME); + protected static final Option OPTION_SEGMENT_NAME = OptionBuilder.withArgName(BatchConstants.ARG_SEGMENT_NAME).hasArg().isRequired(true).withDescription("Cube segment name").create(BatchConstants.ARG_SEGMENT_NAME); protected static final Option OPTION_INPUT_PATH = OptionBuilder.withArgName(BatchConstants.ARG_INPUT).hasArg().isRequired(true).withDescription("Input path").create(BatchConstants.ARG_INPUT); protected static final Option OPTION_INPUT_FORMAT = OptionBuilder.withArgName(BatchConstants.ARG_INPUT_FORMAT).hasArg().isRequired(false).withDescription("Input format").create(BatchConstants.ARG_INPUT_FORMAT); protected static final Option OPTION_OUTPUT_PATH = OptionBuilder.withArgName(BatchConstants.ARG_OUTPUT).hasArg().isRequired(true).withDescription("Output path").create(BatchConstants.ARG_OUTPUT); @@ -90,6 +90,22 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { protected static final Option OPTION_STATISTICS_OUTPUT = OptionBuilder.withArgName(BatchConstants.ARG_STATS_OUTPUT).hasArg().isRequired(false).withDescription("Statistics output").create(BatchConstants.ARG_STATS_OUTPUT); protected static final Option OPTION_STATISTICS_SAMPLING_PERCENT = OptionBuilder.withArgName(BatchConstants.ARG_STATS_SAMPLING_PERCENT).hasArg().isRequired(false).withDescription("Statistics sampling percentage").create(BatchConstants.ARG_STATS_SAMPLING_PERCENT); + private static final String MAP_REDUCE_CLASSPATH = "mapreduce.application.classpath"; + + private static final String KYLIN_HIVE_DEPENDENCY_JARS = "[^,]*hive-exec[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-metastore[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-hcatalog-core[0-9.-]+[^,]*?\\.jar"; + + protected static void runJob(Tool job, String[] args) { + try { + int exitCode = ToolRunner.run(job, args); + System.exit(exitCode); + } catch (Exception e) { + e.printStackTrace(System.err); + System.exit(5); + } + } + + // ============================================================================ + protected String name; protected boolean isAsync = false; protected OptionsHelper optionsHelper = new OptionsHelper(); @@ -137,38 +153,6 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { return retVal; } - protected static void runJob(Tool job, String[] args) { - try { - int exitCode = ToolRunner.run(job, args); - System.exit(exitCode); - } catch (Exception e) { - e.printStackTrace(System.err); - System.exit(5); - } - } - - private static final String KYLIN_HIVE_DEPENDENCY_JARS = "[^,]*hive-exec[0-9.-]+\\.jar|[^,]*hive-metastore[0-9.-]+\\.jar|[^,]*hive-hcatalog-core[0-9.-]+\\.jar"; - - String filterKylinHiveDependency(String kylinHiveDependency) { - if (StringUtils.isBlank(kylinHiveDependency)) - return ""; - - StringBuilder jarList = new StringBuilder(); - - Pattern hivePattern = Pattern.compile(KYLIN_HIVE_DEPENDENCY_JARS); - Matcher matcher = hivePattern.matcher(kylinHiveDependency); - - while (matcher.find()) { - if (jarList.length() > 0) - jarList.append(","); - jarList.append(matcher.group()); - } - - return jarList.toString(); - } - - private static final String MAP_REDUCE_CLASSPATH = "mapreduce.application.classpath"; - protected void setJobClasspath(Job job) { String jarPath = KylinConfig.getInstanceFromEnv().getKylinJobJarPath(); File jarFile = new File(jarPath); @@ -254,6 +238,24 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { setJobTmpJarsAndFiles(job, kylinDependency.toString()); } + private String filterKylinHiveDependency(String kylinHiveDependency) { + if (StringUtils.isBlank(kylinHiveDependency)) + return ""; + + StringBuilder jarList = new StringBuilder(); + + Pattern hivePattern = Pattern.compile(KYLIN_HIVE_DEPENDENCY_JARS); + Matcher matcher = hivePattern.matcher(kylinHiveDependency); + + while (matcher.find()) { + if (jarList.length() > 0) + jarList.append(","); + jarList.append(matcher.group()); + } + + return jarList.toString(); + } + private void setJobTmpJarsAndFiles(Job job, String kylinDependency) { if (StringUtils.isBlank(kylinDependency)) return;
