AMBARI-14839 - DEA is not enabled due to configuration conflict and history log directory can not be changed (Jeff Zhang via jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7a0e1a07 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7a0e1a07 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7a0e1a07 Branch: refs/heads/2.2.1-maint Commit: 7a0e1a07078d5d5d01b73f323c0b1789e78b0b1f Parents: c2d27f3 Author: Jonathan Hurley <jhur...@hortonworks.com> Authored: Fri Jan 29 11:27:35 2016 -0500 Committer: Mahadev Konar <maha...@apache.org> Committed: Sun Jan 31 18:14:14 2016 -0800 ---------------------------------------------------------------------- .../SPARK/1.2.0.2.2/package/scripts/params.py | 2 +- .../SPARK/configuration/spark-defaults.xml | 4 +-- .../configuration/spark-thrift-sparkconf.xml | 31 ++++++++++++++++---- .../stacks/HDP/2.4/upgrades/config-upgrade.xml | 15 ---------- 4 files changed, 29 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/7a0e1a07/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py index a681a5c..4fd1d6b 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py @@ -79,7 +79,7 @@ hive_user = status_params.hive_user spark_group = status_params.spark_group user_group = status_params.user_group spark_hdfs_user_dir = format("/user/{spark_user}") -spark_history_dir = 'hdfs:///spark-history' +spark_history_dir = default('/configurations/spark-defaults/spark.history.fs.logDirectory', "hdfs:///spark-history") spark_history_server_pid_file = status_params.spark_history_server_pid_file spark_thrift_server_pid_file = status_params.spark_thrift_server_pid_file http://git-wip-us.apache.org/repos/asf/ambari/blob/7a0e1a07/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-defaults.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-defaults.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-defaults.xml index d8af790..1a6552f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-defaults.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-defaults.xml @@ -31,7 +31,7 @@ </property> <property> <name>spark.history.fs.logDirectory</name> - <value>{{spark_history_dir}}</value> + <value>hdfs:///spark-history</value> <description> Base directory for history spark application log. </description> @@ -45,7 +45,7 @@ </property> <property> <name>spark.eventLog.dir</name> - <value>{{spark_history_dir}}</value> + <value>hdfs:///spark-history</value> <description> Base directory in which Spark events are logged, if spark.eventLog.enabled is true. </description> http://git-wip-us.apache.org/repos/asf/ambari/blob/7a0e1a07/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-thrift-sparkconf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-thrift-sparkconf.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-thrift-sparkconf.xml index 3b13496..2dbfe51 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-thrift-sparkconf.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-thrift-sparkconf.xml @@ -75,14 +75,17 @@ <property> <name>spark.history.fs.logDirectory</name> <value>{{spark_history_dir}}</value> + <final>true</final> <description> - Base directory for history spark application log. + Base directory for history spark application log. It is the same value + as in spark-defaults.xml. </description> </property> <property> <name>spark.eventLog.enabled</name> <value>true</value> + <final>true</final> <description> Whether to log Spark events, useful for reconstructing the Web UI after the application has finished. </description> @@ -91,8 +94,10 @@ <property> <name>spark.eventLog.dir</name> <value>{{spark_history_dir}}</value> + <final>true</final> <description> - Base directory in which Spark events are logged, if spark.eventLog.enabled is true. + Base directory in which Spark events are logged, if spark.eventLog.enabled is true. It is the same value + as in spark-defaults.xml. </description> </property> @@ -138,10 +143,26 @@ </property> <property> - <name>spark.executor.instances</name> - <value>2</value> + <name>spark.dynamicAllocation.initialExecutors</name> + <value>0</value> <description> - The number of executor. + Initial number of executors to run if dynamic allocation is enabled. + </description> + </property> + + <property> + <name>spark.dynamicAllocation.maxExecutors</name> + <value>10</value> + <description> + Upper bound for the number of executors if dynamic allocation is enabled. + </description> + </property> + + <property> + <name>spark.dynamicAllocation.minExecutors</name> + <value>0</value> + <description> + Lower bound for the number of executors if dynamic allocation is enabled. </description> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/7a0e1a07/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml index 5949486..ee06577 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml @@ -172,9 +172,6 @@ <type>spark-defaults</type> <transfer operation="delete" delete-key="spark.yarn.services" /> <set key="spark.history.provider" value="org.apache.spark.deploy.history.FsHistoryProvider"/> - <set key="spark.history.fs.logDirectory" value="{{spark_history_dir}}"/> - <set key="spark.eventLog.enabled" value="true"/> - <set key="spark.eventLog.dir" value="{{spark_history_dir}}"/> </definition> </changes> </component> @@ -190,18 +187,6 @@ <transfer operation="delete" delete-key="spark.yarn.submit.file.replication" /> <transfer operation="delete" delete-key="spark.yarn.preserve.staging.files" /> <transfer operation="delete" delete-key="spark.yarn.max.executor.failures" /> - <set key="spark.history.provider" value="org.apache.spark.deploy.history.FsHistoryProvider"/> - <set key="spark.history.fs.logDirectory" value="{{spark_history_dir}}"/> - <set key="spark.eventLog.enabled" value="true"/> - <set key="spark.eventLog.dir" value="{{spark_history_dir}}"/> - <set key="spark.master" value="{{spark_thrift_master}}"/> - <set key="spark.scheduler.allocation.file" value="{{spark_conf}}/spark-thrift-fairscheduler.xml"/> - <set key="spark.scheduler.mode" value="FAIR"/> - <set key="spark.shuffle.service.enabled" value="true"/> - <set key="spark.dynamicAllocation.enabled" value="true"/> - <set key="spark.executor.instances" value="2"/> - <set key="spark.yarn.am.memory" value="512m"/> - <set key="spark.executor.memory" value="1g"/> </definition> </changes> </component>