AMBARI-21768. Spark History Server uses wrong log dir. (Attila Doroszlai via swagle)
(cherry picked from commit d1df4d9aa658975a82b5cb5e16be893d36268c87) Change-Id: I94b918c2e186a2de03583ee8fd5a4e9bdb81fe1f Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/92770907 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/92770907 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/92770907 Branch: refs/heads/2.5-maint Commit: 92770907f7dbc7dbd3303167d463c4f838048d5c Parents: a4f7784 Author: Siddharth Wagle <[email protected]> Authored: Mon Aug 21 13:53:26 2017 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Mon Aug 21 17:09:22 2017 -0700 ---------------------------------------------------------------------- .../BigInsights/4.2/upgrades/config-upgrade.xml | 16 ++++++++++++++++ .../4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml | 6 ++++++ 2 files changed, 22 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/92770907/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml index ad20bf9..02d3a01 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml @@ -372,6 +372,22 @@ </definition> </changes> </component> + <component name="SPARK_JOBHISTORYSERVER"> + <changes> + <definition xsi:type="configure" id="biginsights_4_2_spark_history_defaults" summary="Update spark-defaults for History Server"> + <type>spark-defaults</type> + <transfer operation="copy" to-key="spark.history.fs.logDirectory" from-type="spark-defaults" from-key="spark.eventLog.dir" if-type="spark-defaults" if-key="spark.eventLog.dir" if-key-state="present" /> + <replace key="spark.eventLog.dir" find="hdfs:///spark-history" replace-with="hdfs:///iop/apps/4.2.0.0/spark/logs/history-server" /> + <replace key="spark.history.fs.logDirectory" find="hdfs:///spark-history" replace-with="hdfs:///iop/apps/4.2.0.0/spark/logs/history-server" /> + <insert key="spark.eventLog.dir" value="hdfs://" insert-type="prepend" /> + <insert key="spark.history.fs.logDirectory" value="hdfs://" insert-type="prepend" /> + </definition> + <definition xsi:type="configure" id="biginsights_4_2_spark_history_env" summary="Update spark-env for History Server"> + <type>spark-env</type> + <replace key="content" find="-Dspark.history.fs.logDirectory={{spark_eventlog_dir_default}}" replace-with="" /> + </definition> + </changes> + </component> </service> <service name="KNOX"> http://git-wip-us.apache.org/repos/asf/ambari/blob/92770907/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml index b8c23bb..c831326 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml @@ -316,6 +316,12 @@ <execute-stage service="SPARK" component="SPARK_THRIFTSERVER" title="Apply config changes for Spark Thrift server"> <task xsi:type="configure" id="biginsights_4_2_spark_env" /> </execute-stage> + <execute-stage service="SPARK" component="SPARK_JOBHISTORYSERVER" title="Apply config changes for Spark History Server"> + <task xsi:type="configure" id="biginsights_4_2_spark_history_defaults" /> + </execute-stage> + <execute-stage service="SPARK" component="SPARK_JOBHISTORYSERVER" title="Apply config changes for Spark History Server"> + <task xsi:type="configure" id="biginsights_4_2_spark_history_env" /> + </execute-stage> <!-- OOZIE --> <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie server">
