Repository: kylin Updated Branches: refs/heads/v1.5.2-release e80b13ff1 -> a63f959eb
KYLIN-1566 use a separate kylin_job_conf.xml for in-mem cubing Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a63f959e Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a63f959e Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a63f959e Branch: refs/heads/v1.5.2-release Commit: a63f959eb33a2638733817ac796a1f7c8cf89608 Parents: e80b13f Author: shaofengshi <shaofeng...@apache.org> Authored: Wed May 18 09:52:32 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Wed May 18 10:39:35 2016 +0800 ---------------------------------------------------------------------- .../kylin/job/engine/JobEngineConfig.java | 9 +- .../kylin/engine/mr/JobBuilderSupport.java | 2 +- .../sandbox/kylin_job_conf_inmem.xml | 99 ++++++++++++++++++++ .../cardinality/HiveColumnCardinalityJob.java | 2 +- 4 files changed, 105 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/a63f959e/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java index baa3bfe..64e801e 100644 --- a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java +++ b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java @@ -77,13 +77,12 @@ public class JobEngineConfig { * @return the job config file path * @throws IOException */ - public String getHadoopJobConfFilePath(String jobType, String capacity) throws IOException { - String suffix; + public String getHadoopJobConfFilePath(String jobType) throws IOException { + String suffix = null; if(!StringUtils.isEmpty(jobType)) { - suffix = jobType + "_" + capacity; - } else { - suffix = capacity; + suffix = jobType; } + String path = getHadoopJobConfFilePath(suffix, true); if (StringUtils.isEmpty(path)) { path = getHadoopJobConfFilePath(jobType, true); http://git-wip-us.apache.org/repos/asf/kylin/blob/a63f959e/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java index 841c402..151d867 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java @@ -171,7 +171,7 @@ public class JobBuilderSupport { public void appendMapReduceParameters(StringBuilder buf, String jobType, String capacity) { try { - String jobConf = config.getHadoopJobConfFilePath(jobType, capacity); + String jobConf = config.getHadoopJobConfFilePath(jobType); if (jobConf != null && jobConf.length() > 0) { buf.append(" -conf ").append(jobConf); } http://git-wip-us.apache.org/repos/asf/kylin/blob/a63f959e/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml b/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml new file mode 100644 index 0000000..5f2cfba --- /dev/null +++ b/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml @@ -0,0 +1,99 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration> + + <property> + <name>mapreduce.job.split.metainfo.maxsize</name> + <value>-1</value> + <description>The maximum permissible size of the split metainfo file. + The JobTracker won't attempt to read split metainfo files bigger than + the configured value. No limits if set to -1. + </description> + </property> + + <property> + <name>mapreduce.map.maxattempts</name> + <value>2</value> + </property> + + <!-- + <property> + <name>mapred.compress.map.output</name> + <value>true</value> + <description>Compress map outputs</description> + </property> + + <property> + <name>mapred.map.output.compression.codec</name> + <value>org.apache.hadoop.io.compress.SnappyCodec</value> + <description>The compression codec to use for map outputs + </description> + </property> + + <property> + <name>mapred.output.compress</name> + <value>true</value> + <description>Compress the output of a MapReduce job</description> + </property> + + <property> + <name>mapred.output.compression.codec</name> + <value>org.apache.hadoop.io.compress.SnappyCodec</value> + <description>The compression codec to use for job outputs + </description> + </property> + + <property> + <name>mapred.output.compression.type</name> + <value>BLOCK</value> + <description>The compression type to use for job outputs</description> + </property> +--> + + <property> + <name>mapreduce.job.max.split.locations</name> + <value>2000</value> + <description>No description</description> + </property> + + <property> + <name>dfs.replication</name> + <value>2</value> + <description>Block replication</description> + </property> + + <!--Additional config for in-mem cubing, giving mapper more memory --> + <property> + <name>mapreduce.map.memory.mb</name> + <value>512</value> + <description></description> + </property> + + <property> + <name>mapreduce.map.java.opts</name> + <value>-Xmx384m</value> + <description></description> + </property> + + <property> + <name>mapreduce.task.io.sort.mb</name> + <value>100</value> + <description></description> + </property> + +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/a63f959e/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java index e6aca4e..c613ef6 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java @@ -71,7 +71,7 @@ public class HiveColumnCardinalityJob extends AbstractHadoopJob { Configuration conf = getConf(); JobEngineConfig jobEngineConfig = new JobEngineConfig(KylinConfig.getInstanceFromEnv()); - conf.addResource(new Path(jobEngineConfig.getHadoopJobConfFilePath(null, null))); + conf.addResource(new Path(jobEngineConfig.getHadoopJobConfFilePath(null))); job = Job.getInstance(conf, jobName);
