Repository: hive Updated Branches: refs/heads/master 462b47a01 -> 442a1d728
HIVE-13895: HoS start-up overhead in yarn-client mode (Rui reviewed by Xuefu and Szehon) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/442a1d72 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/442a1d72 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/442a1d72 Branch: refs/heads/master Commit: 442a1d728f3293d09c35a75a848bbd6d5c170adb Parents: 462b47a Author: Rui Li <[email protected]> Authored: Thu Jun 2 11:17:00 2016 +0800 Committer: Rui Li <[email protected]> Committed: Thu Jun 2 11:17:43 2016 +0800 ---------------------------------------------------------------------- .../hive/ql/exec/spark/HiveSparkClientFactory.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/442a1d72/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java index b36c60e..b1df1d6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java @@ -53,7 +53,7 @@ public class HiveSparkClientFactory { private static final String SPARK_DEFAULT_APP_NAME = "Hive on Spark"; private static final String SPARK_DEFAULT_SERIALIZER = "org.apache.spark.serializer.KryoSerializer"; private static final String SPARK_DEFAULT_REFERENCE_TRACKING = "false"; - private static final String SPARK_YARN_REPORT_INTERVAL = "spark.yarn.report.interval"; + private static final String SPARK_WAIT_APP_COMPLETE = "spark.yarn.submit.waitAppCompletion"; public static HiveSparkClient createHiveSparkClient(HiveConf hiveconf) throws Exception { Map<String, String> sparkConf = initiateSparkConf(hiveconf); @@ -188,12 +188,9 @@ public class HiveSparkClientFactory { } } - //The application reports tend to spam the hive logs. This is controlled by spark, and the default seems to be 1s. - //If it is not specified, set it to a much higher number. It can always be overriden by user. - String sparkYarnReportInterval = sparkConf.get(SPARK_YARN_REPORT_INTERVAL); - if (sparkMaster.startsWith("yarn") && sparkYarnReportInterval == null) { - //the new version of spark also takes time-units, but old versions do not. - sparkConf.put(SPARK_YARN_REPORT_INTERVAL, "60000"); + // Disable it to avoid verbose app state report in yarn-cluster mode + if (sparkMaster.equals("yarn-cluster") && sparkConf.get(SPARK_WAIT_APP_COMPLETE) == null) { + sparkConf.put(SPARK_WAIT_APP_COMPLETE, "false"); } return sparkConf;
