[kylin] branch master updated: KYLIN-4379: use config overwrite for all MR job

[shaofengshi]

This is an automated email from the ASF dual-hosted git repository.

shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git


The following commit(s) were added to refs/heads/master by this push:
     new df26d77  KYLIN-4379: use config overwrite for all MR job
df26d77 is described below

commit df26d77d2007ad7a64b639c3c1047fc81c28de86
Author: Zhou Kang <zhouka...@xiaomi.com>
AuthorDate: Mon Feb 17 13:40:36 2020 +0800

    KYLIN-4379: use config overwrite for all MR job
---
 .../engine/mr/common/MapReduceExecutable.java      | 26 +++++++++++++---------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
index 260c9ee..a33f171 100755
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
@@ -489,18 +489,24 @@ public class MapReduceExecutable extends 
AbstractExecutable {
             conf.addResource(new Path(confFile));
         }
 
-        if (StringUtils.isNotBlank(cubeName)) {
-            for (Map.Entry<String, String> entry : 
CubeManager.getInstance(config).getCube(cubeName).getConfig()
-                    .getMRConfigOverride().entrySet()) {
+        KylinConfig configOverride;
+        if (cubeName != null) {
+            configOverride = 
CubeManager.getInstance(config).getCube(cubeName).getConfig();
+        } else {
+            configOverride = config;
+        }
+
+        for (Map.Entry<String, String> entry : 
configOverride.getMRConfigOverride().entrySet()) {
+            conf.set(entry.getKey(), entry.getValue());
+        }
+        if (conf.get("mapreduce.job.is-mem-hungry") != null
+                && 
Boolean.parseBoolean(conf.get("mapreduce.job.is-mem-hungry"))) {
+            for (Map.Entry<String, String> entry : 
configOverride.getMemHungryConfigOverride().entrySet()) {
                 conf.set(entry.getKey(), entry.getValue());
             }
-            if (conf.get("mapreduce.job.is-mem-hungry") != null
-                    && 
Boolean.parseBoolean(conf.get("mapreduce.job.is-mem-hungry"))) {
-                for (Map.Entry<String, String> entry : 
CubeManager.getInstance(config).getCube(cubeName).getConfig()
-                        .getMemHungryConfigOverride().entrySet()) {
-                    conf.set(entry.getKey(), entry.getValue());
-                }
-            }
+        }
+
+        if (StringUtils.isNotBlank(cubeName)) {
             remainingArgs.add("-" + BatchConstants.ARG_CUBE_NAME);
             remainingArgs.add(cubeName);
         }