KYLIN-1226 apply in-mem conf only to in-mem mr job
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0232f17e Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0232f17e Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0232f17e Branch: refs/heads/1.5.x-HBase1.1.3 Commit: 0232f17ed74a3354c7ffd6647d159a8581626861 Parents: 5e13bba Author: Li Yang <[email protected]> Authored: Fri Mar 11 15:03:40 2016 +0800 Committer: Li Yang <[email protected]> Committed: Fri Mar 11 15:04:19 2016 +0800 ---------------------------------------------------------------------- build/conf/kylin_job_conf.xml | 14 ------------- .../apache/kylin/common/KylinConfigBase.java | 21 +++++++++++++++++++- .../kylin/engine/mr/steps/InMemCuboidJob.java | 11 ++++++++++ 3 files changed, 31 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/0232f17e/build/conf/kylin_job_conf.xml ---------------------------------------------------------------------- diff --git a/build/conf/kylin_job_conf.xml b/build/conf/kylin_job_conf.xml index 052021c..3ecaeda 100644 --- a/build/conf/kylin_job_conf.xml +++ b/build/conf/kylin_job_conf.xml @@ -55,20 +55,6 @@ <description>Block replication</description> </property> - <!--Properties for calculating cube by splits (in-mem), with which each Mapper need more mem to hold a full cube segment --> - <property> - <name>mapreduce.map.java.opts</name> - <value>-Xmx2500m</value> - </property> - <property> - <name>mapreduce.map.memory.mb</name> - <value>3072</value> - </property> - <property> - <name>mapreduce.task.io.sort.mb</name> - <value>200</value> - </property> - <property> <name>mapred.task.timeout</name> <value>3600000</value> http://git-wip-us.apache.org/repos/asf/kylin/blob/0232f17e/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 487f78e..6a47321 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.Serializable; +import java.util.Map; import java.util.Properties; import java.util.SortedSet; import java.util.regex.Matcher; @@ -33,6 +34,7 @@ import org.apache.kylin.common.util.CliCommandExecutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; @SuppressWarnings("serial") @@ -476,11 +478,28 @@ public class KylinConfigBase implements Serializable { } public int getCubingInMemSamplingPercent() { - int percent = Integer.parseInt(this.getOptional("kylin.job.cubing.inMem.sampling.percent", "30")); + int percent = Integer.parseInt(this.getOptional("kylin.job.cubing.inmem.sampling.percent", "30")); percent = Math.max(percent, 1); percent = Math.min(percent, 100); return percent; } + + public Map<String, String> getCubingInMemMRJobConfOverride() { + String str = getOptional("kylin.job.cubing.inmem.mrjob_conf_override", // + "mapreduce.map.java.opts=-Xmx2700m; mapreduce.map.memory.mb=3072; mapreduce.task.io.sort.mb=200"); + Map<String, String> result = Maps.newHashMap(); + for (String pair : str.split(";")) { + int cut = pair.indexOf('='); + if (cut < 0) + continue; + String k = pair.substring(0, cut).trim(); + String v = pair.substring(cut + 1).trim(); + if (k.isEmpty() || v.isEmpty()) + continue; + result.put(k, v); + } + return result; + } public String getHbaseDefaultCompressionCodec() { return getOptional("kylin.hbase.default.compression.codec", ""); http://git-wip-us.apache.org/repos/asf/kylin/blob/0232f17e/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java index d197252..2319451 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java @@ -20,8 +20,10 @@ package org.apache.kylin.engine.mr.steps; import java.io.IOException; import java.util.Map; +import java.util.Map.Entry; import org.apache.commons.cli.Options; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; @@ -97,6 +99,9 @@ public class InMemCuboidJob extends AbstractHadoopJob { job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME)); logger.info("Starting: " + job.getJobName()); + + // some special tuning for in-mem MR job + overrideJobConf(job.getConfiguration(), config); setJobClasspath(job); @@ -143,6 +148,12 @@ public class InMemCuboidJob extends AbstractHadoopJob { } } + private void overrideJobConf(Configuration jobConf, KylinConfig kylinConfig) { + for (Entry<String, String> entry : kylinConfig.getCubingInMemMRJobConfOverride().entrySet()) { + jobConf.set(entry.getKey(), entry.getValue()); + } + } + private int calculateReducerNum(CubeSegment cubeSeg) throws IOException { KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
