[kylin] branch master updated: KYLIN-3866 Whether to set mapreduce.application.classpath is determined by the user

[shaofengshi]

This is an automated email from the ASF dual-hosted git repository.

shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git


The following commit(s) were added to refs/heads/master by this push:
     new d23d2bd  KYLIN-3866 Whether to set mapreduce.application.classpath is 
determined by the user
d23d2bd is described below

commit d23d2bddab3263d39059492b91f71d8091198057
Author: Guangxu Cheng <gxch...@apache.org>
AuthorDate: Mon Mar 11 17:37:06 2019 +0800

    KYLIN-3866 Whether to set mapreduce.application.classpath is determined by 
the user
    
    Signed-off-by: shaofengshi <shaofeng...@apache.org>
---
 .../java/org/apache/kylin/common/KylinConfigBase.java   |  5 +++++
 .../kylin/engine/mr/common/AbstractHadoopJob.java       | 17 ++++++++++-------
 2 files changed, 15 insertions(+), 7 deletions(-)

diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 3edd458..82f79eb 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1332,6 +1332,11 @@ public abstract class KylinConfigBase implements 
Serializable {
         return 
Integer.parseInt(getOptional("kylin.engine.mr.yarn-check-interval-seconds", 
"10"));
     }
 
+
+    public boolean isUseLocalClasspathEnabled() {
+        return 
Boolean.parseBoolean(getOptional("kylin.engine.mr.use-local-classpath", TRUE));
+    }
+
     // 
============================================================================
     // ENGINE.SPARK
     // 
============================================================================
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index ca92dfd..76286e3 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -214,15 +214,18 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
         String kylinKafkaDependency = 
System.getProperty("kylin.kafka.dependency");
 
         Configuration jobConf = job.getConfiguration();
-        String classpath = jobConf.get(MAP_REDUCE_CLASSPATH);
-        if (classpath == null || classpath.length() == 0) {
-            logger.info("Didn't find " + MAP_REDUCE_CLASSPATH
+
+        if (kylinConf.isUseLocalClasspathEnabled()) {
+            String classpath = jobConf.get(MAP_REDUCE_CLASSPATH);
+            if (classpath == null || classpath.length() == 0) {
+                logger.info("Didn't find " + MAP_REDUCE_CLASSPATH
                     + " in job configuration, will run 'mapred classpath' to 
get the default value.");
-            classpath = getDefaultMapRedClasspath();
-            logger.info("The default mapred classpath is: " + classpath);
-        }
+                classpath = getDefaultMapRedClasspath();
+                logger.info("The default mapred classpath is: " + classpath);
+            }
 
-        jobConf.set(MAP_REDUCE_CLASSPATH, classpath);
+            jobConf.set(MAP_REDUCE_CLASSPATH, classpath);
+        }
         logger.trace("Hadoop job classpath is: " + 
job.getConfiguration().get(MAP_REDUCE_CLASSPATH));
 
         /*