Repository: ambari Updated Branches: refs/heads/trunk 043d6c06c -> 19da5823c
AMBARI-20126. Add support for Spark2 upgrade from HDP-2.5 (dgrinenko via echekankiy) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/19da5823 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/19da5823 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/19da5823 Branch: refs/heads/trunk Commit: 19da5823c8adc8cd18048040c93f43941c0e568f Parents: 043d6c0 Author: Eugene Chekanskiy <[email protected]> Authored: Fri Feb 24 11:18:33 2017 +0200 Committer: Eugene Chekanskiy <[email protected]> Committed: Fri Feb 24 11:18:33 2017 +0200 ---------------------------------------------------------------------- .../2.0.0/package/scripts/job_history_server.py | 2 +- .../2.0.0/package/scripts/livy2_server.py | 2 +- .../2.0.0/package/scripts/spark_client.py | 2 +- .../package/scripts/spark_thrift_server.py | 9 ++-- .../stacks/HDP/2.5/upgrades/config-upgrade.xml | 11 +++++ .../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 50 +++++++++++++++++++ .../stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 39 +++++++++++++++ .../HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml | 51 ++++++++++++++++++++ .../stacks/HDP/2.6/upgrades/upgrade-2.6.xml | 41 ++++++++++++++++ 9 files changed, 199 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/job_history_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/job_history_server.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/job_history_server.py index 154c83d..2631b49 100755 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/job_history_server.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/job_history_server.py @@ -41,7 +41,7 @@ class JobHistoryServer(Script): self.install_packages(env) - def configure(self, env, upgrade_type=None): + def configure(self, env, upgrade_type=None, config_dir=None): import params env.set_params(params) http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py index 8c66998..cb4f5ee 100644 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py @@ -44,7 +44,7 @@ class LivyServer(Script): self.install_packages(env) - def configure(self, env, upgrade_type=None): + def configure(self, env, upgrade_type=None, config_dir=None): import params env.set_params(params) http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_client.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_client.py index 2c19b88..563b7e9 100755 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_client.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_client.py @@ -34,7 +34,7 @@ class SparkClient(Script): self.install_packages(env) self.configure(env) - def configure(self, env, upgrade_type=None): + def configure(self, env, upgrade_type=None, config_dir=None): import params env.set_params(params) http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_thrift_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_thrift_server.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_thrift_server.py index 426c05c..72307cb 100755 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_thrift_server.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_thrift_server.py @@ -40,7 +40,7 @@ class SparkThriftServer(Script): self.install_packages(env) - def configure(self, env ,upgrade_type=None): + def configure(self, env, upgrade_type=None, config_dir=None): import params env.set_params(params) setup_spark(env, 'server', upgrade_type = upgrade_type, action = 'config') @@ -69,10 +69,9 @@ class SparkThriftServer(Script): import params env.set_params(params) - if params.version and check_stack_feature(StackFeature.SPARK2_THRIFTSERVER, params.version): - Logger.info("Executing Spark2 Thrift Server Stack Upgrade pre-restart") - conf_select.select(params.stack_name, "spark2", params.version) - stack_select.select("spark2-thriftserver", params.version) + Logger.info("Executing Spark2 Thrift Server Stack Upgrade pre-restart") + conf_select.select(params.stack_name, "spark2", params.version) + stack_select.select("spark2-thriftserver", params.version) def get_log_folder(self): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml index 3069b34..da334be 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml @@ -94,6 +94,17 @@ </component> </service> + <service name="SPARK2"> + <component name="SPARK2_CLIENT"> + <changes> + <definition xsi:type="configure" id="hdp_2_5_0_0_spark2_yarn_queue"> + <type>spark2-defaults</type> + <set key="spark.yarn.queue" value="default" if-type="spark-defaults" if-key="spark.yarn.queue" if-key-state="absent"/> + </definition> + </changes> + </component> + </service> + <service name="TEZ"> <component name="TEZ_CLIENT"> <changes> http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml index 9516b6a..d396033 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml @@ -110,6 +110,11 @@ <component>LIVY_SERVER</component> </service> + <service name="SPARK2"> + <component>SPARK2_JOBHISTORYSERVER</component> + <component>SPARK2_THRIFTSERVER</component> + </service> + <service name="ZEPPELIN"> <component>ZEPPELIN_MASTER</component> </service> @@ -538,6 +543,12 @@ <summary>Add queue customization property</summary> </task> </execute-stage> + + <execute-stage service="SPARK2" component="SPARK2_CLIENT" title="Apply config changes for Spark"> + <task xsi:type="configure" id="hdp_2_5_0_0_spark2_yarn_queue"> + <summary>Add queue customization property</summary> + </task> + </execute-stage> </group> <!-- @@ -795,6 +806,17 @@ </service> </group> + <group xsi:type="restart" name="SPARK2" title="Spark2"> + <service-check>false</service-check> + <skippable>true</skippable> + <supports-auto-skip-failure>false</supports-auto-skip-failure> + <parallel-scheduler/> + <service name="SPARK2"> + <component>SPARK2_JOBHISTORYSERVER</component> + <component>SPARK2_THRIFTSERVER</component> + </service> + </group> + <group xsi:type="restart" name="SPARK_CLIENTS" title="Spark Clients"> <service-check>false</service-check> <skippable>true</skippable> @@ -804,6 +826,15 @@ </service> </group> + <group xsi:type="restart" name="SPARK2_CLIENTS" title="Spark2 Clients"> + <service-check>false</service-check> + <skippable>true</skippable> + <parallel-scheduler/> + <service name="SPARK2"> + <component>SPARK2_CLIENT</component> + </service> + </group> + <group xsi:type="restart" name="ZEPPELIN" title="Zeppelin"> <service-check>false</service-check> <skippable>true</skippable> @@ -860,6 +891,7 @@ <priority> <service>HIVE</service> <service>SPARK</service> + <service>SPARK2</service> <service>SLIDER</service> <service>ATLAS</service> <service>ZEPPELIN</service> @@ -1275,6 +1307,24 @@ </component> </service> + <service name="SPARK2"> + <component name="SPARK2_JOBHISTORYSERVER"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + <component name="SPARK2_THRIFTSERVER"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + <component name="SPARK2_CLIENT"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + </service> + <service name="ZEPPELIN"> <component name="ZEPPELIN_MASTER"> <upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml index 04d603b..c01d11e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml @@ -259,6 +259,16 @@ </service> </group> + <group name="SPARK2" title="Spark2"> + <skippable>true</skippable> + <supports-auto-skip-failure>false</supports-auto-skip-failure> + <service-check>false</service-check> + <service name="SPARK2"> + <component>SPARK2_JOBHISTORYSERVER</component> + <component>SPARK2_THRIFTSERVER</component> + </service> + </group> + <group name="ZEPPELIN" title="Zeppelin"> <skippable>true</skippable> <supports-auto-skip-failure>false</supports-auto-skip-failure> @@ -275,6 +285,13 @@ </service> </group> + <group name="SPARK2_CLIENTS" title="Spark2 Clients"> + <skippable>true</skippable> + <service name="SPARK2"> + <component>SPARK2_CLIENT</component> + </service> + </group> + <group name="ATLAS" title="Atlas"> <service-check>false</service-check> <skippable>true</skippable> @@ -868,6 +885,28 @@ </component> </service> + <service name="SPARK2"> + <component name="SPARK2_JOBHISTORYSERVER"> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + </component> + <component name="SPARK2_THRIFTSERVER"> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + </component> + <component name="SPARK2_CLIENT"> + <pre-upgrade> + <task xsi:type="configure" id="hdp_2_5_0_0_spark2_yarn_queue"/> + </pre-upgrade> + <pre-downgrade/> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + </component> + </service> + <service name="ZEPPELIN"> <component name="ZEPPELIN_MASTER"> <upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml index 5347f01..921e479 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml @@ -109,6 +109,12 @@ <component>LIVY_SERVER</component> </service> + <service name="SPARK2"> + <component>SPARK2_JOBHISTORYSERVER</component> + <component>SPARK2_THRIFTSERVER</component> + <component>LIVY2_SERVER</component> + </service> + <service name="ZEPPELIN"> <component>ZEPPELIN_MASTER</component> </service> @@ -527,6 +533,18 @@ </service> </group> + <group xsi:type="restart" name="SPARK2" title="Spark2"> + <service-check>false</service-check> + <skippable>true</skippable> + <supports-auto-skip-failure>false</supports-auto-skip-failure> + <parallel-scheduler/> + <service name="SPARK2"> + <component>SPARK2_JOBHISTORYSERVER</component> + <component>SPARK2_THRIFTSERVER</component> + <component>LIVY2_SERVER</component> + </service> + </group> + <group xsi:type="restart" name="SPARK_CLIENTS" title="Spark Clients"> <service-check>false</service-check> <skippable>true</skippable> @@ -536,6 +554,15 @@ </service> </group> + <group xsi:type="restart" name="SPARK2_CLIENTS" title="Spark2 Clients"> + <service-check>false</service-check> + <skippable>true</skippable> + <parallel-scheduler/> + <service name="SPARK2"> + <component>SPARK2_CLIENT</component> + </service> + </group> + <group xsi:type="restart" name="ZEPPELIN" title="Zeppelin"> <service-check>false</service-check> <skippable>true</skippable> @@ -592,6 +619,7 @@ <priority> <service>HIVE</service> <service>SPARK</service> + <service>SPARK2</service> <service>SLIDER</service> <service>ATLAS</service> <service>ZEPPELIN</service> @@ -997,6 +1025,29 @@ </component> </service> + <service name="SPARK2"> + <component name="SPARK2_JOBHISTORYSERVER"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + <component name="SPARK2_THRIFTSERVER"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + <component name="LIVY2_SERVER"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + <component name="SPARK2_CLIENT"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + </service> + <service name="ZEPPELIN"> <component name="ZEPPELIN_MASTER"> <upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml index a4719c1..8669d2b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml @@ -258,6 +258,17 @@ </service> </group> + <group name="SPARK2" title="Spark2"> + <skippable>true</skippable> + <supports-auto-skip-failure>false</supports-auto-skip-failure> + <service-check>false</service-check> + <service name="SPARK2"> + <component>SPARK2_JOBHISTORYSERVER</component> + <component>SPARK2_THRIFTSERVER</component> + <component>LIVY2_SERVER</component> + </service> + </group> + <group name="ZEPPELIN" title="Zeppelin"> <skippable>true</skippable> <supports-auto-skip-failure>false</supports-auto-skip-failure> @@ -274,6 +285,13 @@ </service> </group> + <group name="SPARK2_CLIENTS" title="Spark2 Clients"> + <skippable>true</skippable> + <service name="SPARK2"> + <component>SPARK2_CLIENT</component> + </service> + </group> + <group name="ATLAS" title="Atlas"> <service-check>false</service-check> <skippable>true</skippable> @@ -779,6 +797,29 @@ </component> </service> + <service name="SPARK2"> + <component name="SPARK2_JOBHISTORYSERVER"> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + </component> + <component name="SPARK2_THRIFTSERVER"> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + </component> + <component name="LIVY2_SERVER"> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + </component> + <component name="SPARK2_CLIENT"> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + </component> + </service> + <service name="ZEPPELIN"> <component name="ZEPPELIN_MASTER"> <upgrade>
