HIVE-12811: Name yarn application name more meaning than just "Hive on Spark" (Rui reviewed by Xuefu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e47e1142 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e47e1142 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e47e1142 Branch: refs/heads/master Commit: e47e11424f8867fb88266bda796f5639ccdd6d28 Parents: bb5ad57 Author: Rui Li <[email protected]> Authored: Wed Jan 13 10:41:27 2016 +0800 Committer: Rui Li <[email protected]> Committed: Thu Jan 28 14:51:15 2016 +0800 ---------------------------------------------------------------------- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 3 ++- .../hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/e47e1142/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index c4034a5..b9d4b5e 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2925,7 +2925,8 @@ public class HiveConf extends Configuration { private boolean isSparkRelatedConfig(String name) { boolean result = false; if (name.startsWith("spark")) { // Spark property. - result = true; + // for now we don't support changing spark app name on the fly + result = !name.equals("spark.app.name"); } else if (name.startsWith("yarn")) { // YARN property in Spark on YARN mode. String sparkMaster = get("spark.master"); if (sparkMaster != null && http://git-wip-us.apache.org/repos/asf/hive/blob/e47e1142/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java index 9b2dce3..a832bf6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java @@ -72,7 +72,12 @@ public class HiveSparkClientFactory { // set default spark configurations. sparkConf.put("spark.master", SPARK_DEFAULT_MASTER); - sparkConf.put("spark.app.name", SPARK_DEFAULT_APP_NAME); + final String appNameKey = "spark.app.name"; + String appName = hiveConf.get(appNameKey); + if (appName == null) { + appName = SPARK_DEFAULT_APP_NAME; + } + sparkConf.put(appNameKey, appName); sparkConf.put("spark.serializer", SPARK_DEFAULT_SERIALIZER); sparkConf.put("spark.kryo.referenceTracking", SPARK_DEFAULT_REFERENCE_TRACKING);
