HIVE-12828: Update Spark version to 1.6 (Rui reviewed by Xuefu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/34b41e38 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/34b41e38 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/34b41e38 Branch: refs/heads/master Commit: 34b41e389ab331f1d61b077fbea8cb855775698f Parents: e47e114 Author: Rui Li <[email protected]> Authored: Fri Jan 15 21:24:39 2016 +0800 Committer: Rui Li <[email protected]> Committed: Thu Jan 28 14:51:55 2016 +0800 ---------------------------------------------------------------------- data/conf/spark/yarn-client/hive-site.xml | 17 ++++++++++++++++- pom.xml | 2 +- .../apache/hadoop/hive/shims/Hadoop23Shims.java | 2 ++ 3 files changed, 19 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/34b41e38/data/conf/spark/yarn-client/hive-site.xml ---------------------------------------------------------------------- diff --git a/data/conf/spark/yarn-client/hive-site.xml b/data/conf/spark/yarn-client/hive-site.xml index bc42f0d..f1d9ddc 100644 --- a/data/conf/spark/yarn-client/hive-site.xml +++ b/data/conf/spark/yarn-client/hive-site.xml @@ -231,7 +231,22 @@ <property> <name>spark.executor.memory</name> - <value>512m</value> + <value>1g</value> +</property> + +<property> + <name>spark.yarn.executor.memoryOverhead</name> + <value>0</value> +</property> + +<property> + <name>spark.driver.memory</name> + <value>1g</value> +</property> + +<property> + <name>spark.yarn.driver.memoryOverhead</name> + <value>0</value> </property> <property> http://git-wip-us.apache.org/repos/asf/hive/blob/34b41e38/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 2066518..2d2a3de 100644 --- a/pom.xml +++ b/pom.xml @@ -166,7 +166,7 @@ <ST4.version>4.0.4</ST4.version> <tez.version>0.8.2</tez.version> <super-csv.version>2.2.0</super-csv.version> - <spark.version>1.5.0</spark.version> + <spark.version>1.6.0</spark.version> <scala.binary.version>2.10</scala.binary.version> <scala.version>2.10.4</scala.version> <tempus-fugit.version>1.1</tempus-fugit.version> http://git-wip-us.apache.org/repos/asf/hive/blob/34b41e38/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java ---------------------------------------------------------------------- diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index c03ae35..31060a2 100644 --- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -526,6 +526,8 @@ public class Hadoop23Shims extends HadoopShimsSecure { mr = new MiniSparkOnYARNCluster("sparkOnYarn"); conf.set("fs.defaultFS", nameNode); conf.set("yarn.resourcemanager.scheduler.class", "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"); + // disable resource monitoring, although it should be off by default + conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING, false); configureImpersonation(conf); mr.init(conf); mr.start();
