Repository: kylin Updated Branches: refs/heads/2.x-staging 54e491502 -> 2528111b4
Add conf/kylin_job_conf.xml for HiveColumnCardinalityJob Signed-off-by: shaofengshi <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2528111b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2528111b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2528111b Branch: refs/heads/2.x-staging Commit: 2528111b422a9801f635dc0fd0fa757a70e08f6c Parents: 54e4915 Author: John Zhao <[email protected]> Authored: Tue Feb 2 13:45:50 2016 +0800 Committer: shaofengshi <[email protected]> Committed: Wed Feb 3 10:27:38 2016 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/job/engine/JobEngineConfig.java | 2 +- .../source/hive/cardinality/HiveColumnCardinalityJob.java | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/2528111b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java index b305749..546c033 100644 --- a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java +++ b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java @@ -51,7 +51,7 @@ public class JobEngineConfig { private String getHadoopJobConfFilePath(RealizationCapacity capaticy, boolean appendSuffix) throws IOException { String hadoopJobConfFile; - if (appendSuffix) { + if (capaticy != null && appendSuffix) { hadoopJobConfFile = (HADOOP_JOB_CONF_FILENAME + "_" + capaticy.toString().toLowerCase() + ".xml"); } else { hadoopJobConfFile = (HADOOP_JOB_CONF_FILENAME + ".xml"); http://git-wip-us.apache.org/repos/asf/kylin/blob/2528111b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java index 2b35755..51eb0ef 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java @@ -29,10 +29,13 @@ import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.engine.mr.IMRInput.IMRTableInputFormat; import org.apache.kylin.engine.mr.MRUtil; import org.apache.kylin.engine.mr.common.AbstractHadoopJob; import org.apache.kylin.engine.mr.common.BatchConstants; +import org.apache.kylin.job.engine.JobEngineConfig; +import org.apache.kylin.metadata.model.DataModelDesc; /** * This hadoop job will scan all rows of the hive table and then calculate the cardinality on each column. @@ -65,6 +68,10 @@ public class HiveColumnCardinalityJob extends AbstractHadoopJob { String jobName = JOB_TITLE + getOptionsAsString(); logger.info("Starting: " + jobName); Configuration conf = getConf(); + + JobEngineConfig jobEngineConfig = new JobEngineConfig(KylinConfig.getInstanceFromEnv()); + conf.addResource(jobEngineConfig.getHadoopJobConfFilePath(null)); + job = Job.getInstance(conf, jobName); setJobClasspath(job);
