http://git-wip-us.apache.org/repos/asf/ambari/blob/c58162fe/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql index 2636eef..7cc233b 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql @@ -535,7 +535,6 @@ CREATE TABLE repo_version ( stack_id BIGINT NOT NULL, version VARCHAR(255) NOT NULL, display_name VARCHAR(128) NOT NULL, - upgrade_package VARCHAR(255) NOT NULL, repositories TEXT NOT NULL, PRIMARY KEY(repo_version_id) ); @@ -874,6 +873,8 @@ CREATE TABLE upgrade ( from_version VARCHAR(255) DEFAULT '' NOT NULL, to_version VARCHAR(255) DEFAULT '' NOT NULL, direction VARCHAR(255) DEFAULT 'UPGRADE' NOT NULL, + upgrade_package VARCHAR(255) NOT NULL, + upgrade_type VARCHAR(32) NOT NULL, PRIMARY KEY (upgrade_id), FOREIGN KEY (cluster_id) REFERENCES clusters(cluster_id), FOREIGN KEY (request_id) REFERENCES request(request_id)
http://git-wip-us.apache.org/repos/asf/ambari/blob/c58162fe/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql index d515fa6..77ca0c1 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql @@ -602,7 +602,6 @@ CREATE TABLE ambari.repo_version ( stack_id BIGINT NOT NULL, version VARCHAR(255) NOT NULL, display_name VARCHAR(128) NOT NULL, - upgrade_package VARCHAR(255) NOT NULL, repositories TEXT NOT NULL, PRIMARY KEY(repo_version_id) ); @@ -965,6 +964,8 @@ CREATE TABLE ambari.upgrade ( from_version VARCHAR(255) DEFAULT '' NOT NULL, to_version VARCHAR(255) DEFAULT '' NOT NULL, direction VARCHAR(255) DEFAULT 'UPGRADE' NOT NULL, + upgrade_package VARCHAR(255) NOT NULL, + upgrade_type VARCHAR(32) NOT NULL, PRIMARY KEY (upgrade_id), FOREIGN KEY (cluster_id) REFERENCES ambari.clusters(cluster_id), FOREIGN KEY (request_id) REFERENCES ambari.request(request_id) http://git-wip-us.apache.org/repos/asf/ambari/blob/c58162fe/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql index 1f95f0f..cac2f48 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql @@ -641,7 +641,6 @@ CREATE TABLE repo_version ( stack_id BIGINT NOT NULL, version VARCHAR(255) NOT NULL, display_name VARCHAR(128) NOT NULL, - upgrade_package VARCHAR(255) NOT NULL, repositories VARCHAR(MAX) NOT NULL, PRIMARY KEY CLUSTERED (repo_version_id) ); @@ -989,6 +988,8 @@ CREATE TABLE upgrade ( from_version VARCHAR(255) DEFAULT '' NOT NULL, to_version VARCHAR(255) DEFAULT '' NOT NULL, direction VARCHAR(255) DEFAULT 'UPGRADE' NOT NULL, + upgrade_package VARCHAR(255) NOT NULL, + upgrade_type VARCHAR(32) NOT NULL, PRIMARY KEY CLUSTERED (upgrade_id), FOREIGN KEY (cluster_id) REFERENCES clusters(cluster_id), FOREIGN KEY (request_id) REFERENCES request(request_id) http://git-wip-us.apache.org/repos/asf/ambari/blob/c58162fe/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_upgrade.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_upgrade.py index 610f527..2dc9883 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_upgrade.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_upgrade.py @@ -24,7 +24,7 @@ from resource_management.core.resources.system import Execute class HbaseMasterUpgrade(Script): - def snapshot(self, env): + def take_snapshot(self, env): import params snap_cmd = "echo 'snapshot_all' | {0} shell".format(params.hbase_cmd) @@ -33,5 +33,9 @@ class HbaseMasterUpgrade(Script): Execute(exec_cmd, user=params.hbase_user) + def restore_snapshot(self, env): + import params + print "TODO AMBARI-12698" + if __name__ == "__main__": HbaseMasterUpgrade().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/c58162fe/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py index 1dfb280..93bbc0f 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py @@ -107,9 +107,24 @@ class NameNodeDefault(NameNode): def get_stack_to_component(self): return {"HDP": "hadoop-hdfs-namenode"} + def restore_snapshot(self, env): + """ + Restore the snapshot during a Downgrade. + """ + print "TODO AMBARI-12698" + pass + + def prepare_non_rolling_upgrade(self, env): + print "TODO AMBARI-12698" + pass + def prepare_rolling_upgrade(self, env): namenode_upgrade.prepare_rolling_upgrade() + def finalize_non_rolling_upgrade(self, env): + print "TODO AMBARI-12698" + pass + def finalize_rolling_upgrade(self, env): namenode_upgrade.finalize_rolling_upgrade() http://git-wip-us.apache.org/repos/asf/ambari/blob/c58162fe/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml new file mode 100644 index 0000000..603b1fd --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml @@ -0,0 +1,383 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + + +<upgrade xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> + <target>2.3.*.*</target> + <target-stack>HDP-2.3</target-stack> + <type>NON_ROLLING</type> + <prerequisite-checks> + </prerequisite-checks> + <upgrade-path> + <intermediate-stack version="2.2"/> + <intermediate-stack version="2.3"/> + </upgrade-path> + + <order> + <group xsi:type="cluster" name="PRE_CLUSTER" title="Prepare Upgrade"> + <skippable>true</skippable> + <direction>UPGRADE</direction> + <execute-stage service="YARN" component="RESOURCEMANAGER" title="Stop YARN Queues"> + <task xsi:type="manual"> + <message>Before continuing, please stop all YARN queues. If yarn-site's yarn.resourcemanager.work-preserving-recovery.enabled is set to true, then you can skip this step since the clients will retry on their own.</message> + </task> + </execute-stage> + + <execute-stage service="STORM" component="NIMBUS" title="Stop Storm Topologies"> + <task xsi:type="manual"> + <message>Before continuing, please deactivate and kill any currently running topologies.</message> + </task> + </execute-stage> + </group> + + <group xsi:type="stop" name="Stop High-Level Daemons" title="Stop Daemons for High-Level Services"> + <skippable>true</skippable> + <service-check>false</service-check> + + <service name="FLUME"> + <component>FLUME_HANDLER</component> + </service> + + <service name="STORM"> + <component>DRPC_SERVER</component> + <component>STORM_UI_SERVER</component> + <component>SUPERVISOR</component> + <component>STORM_REST_API</component> + <component>NIMBUS</component> + </service> + + <service name="FALCON"> + <component>FALCON_SERVER</component> + </service> + + <service name="OOZIE"> + <component>OOZIE_SERVER</component> + </service> + + <service name="HIVE"> + <component>WEBHCAT_SERVER</component> + <component>HIVE_SERVER</component> + <component>HIVE_METASTORE</component> + </service> + + <service name="YARN"> + <component>NODEMANAGER</component> + <component>RESOURCEMANAGER</component> + <component>APP_TIMELINE_SERVER</component> + </service> + + <service name="MAPREDUCE2"> + <component>HISTORYSERVER</component> + </service> + </group> + + <group xsi:type="cluster" name="Backups" title="Take Backups"> + <direction>UPGRADE</direction> + <skippable>true</skippable> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Backup Oozie Database"> + <task xsi:type="manual"> + <message>Before continuing, please backup the Oozie Server database on {{oozie-env/oozie_hostname}}.</message> + </task> + </execute-stage> + + <execute-stage service="HIVE" component="HIVE_METASTORE" title="Backup Hive Metastore"> + <task xsi:type="manual"> + <message>Before continuing, please backup the Hive Metastore database located on the following host(s): {{hosts.all}}.</message> + </task> + </execute-stage> + + <execute-stage service="HBASE" component="HBASE_MASTER" title="Snapshot HBASE"> + <task xsi:type="execute" hosts="master"> + <script>scripts/hbase_upgrade.py</script> + <function>take_snapshot</function> + </task> + </execute-stage> + + <execute-stage service="HDFS" component="NAMENODE" title="Snapshot HDFS"> + <task xsi:type="execute" hosts="master"> + <script>scripts/namenode.py</script> + <function>prepare_non_rolling_upgrade</function> + </task> + </execute-stage> + </group> + + <group xsi:type="stop" name="Stop Low-Level Daemons" title="Stop Daemons for Low-Level Services"> + <skippable>true</skippable> + <service-check>false</service-check> + + <service name="HBASE"> + <component>HBASE_REGIONSERVER</component> + <component>HBASE_MASTER</component> + </service> + + <service name="HDFS"> + <component>DATANODE</component> + <component>NAMENODE</component> + <component>SECONDARY_NAMENODE</component> + <component>ZKFC</component> + <component>JOURNALNODE</component> + </service> + + <service name="ZOOKEEPER"> + <component>ZOOKEEPER_SERVER</component> + </service> + </group> + + <group name="Marker for Downgrade" title="Marker for Downgrade"> + <direction>UPGRADE</direction> + <!-- TODO, if the user attempts a downgrade before this step, they can simply abort. --> + </group> + + <group xsi:type="cluster" name="Restore Backups" title="Restore Backups"> + <direction>DOWNGRADE</direction> + <skippable>true</skippable> + + <!-- If the user attempts a downgrade after this point, they will need to restore backups + before starting any of the services. --> + + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Backup Oozie Database"> + <task xsi:type="manual"> + <message>Before continuing, please restore the Oozie Server database on {{hosts.all}}.</message> + </task> + </execute-stage> + + <execute-stage service="HIVE" component="HIVE_METASTORE" title="Backup Hive Metastore"> + <task xsi:type="manual"> + <message>Before continuing, please restore the Hive Metastore database located on the following host(s): {{hosts.all}}.</message> + </task> + </execute-stage> + + <execute-stage service="HBASE" component="HBASE_MASTER" title="Snapshot HBASE"> + <task xsi:type="execute" hosts="master"> + <script>scripts/hbase_upgrade.py</script> + <function>restore_snapshot</function> + </task> + </execute-stage> + + <execute-stage service="HDFS" component="NAMENODE" title="Snapshot HDFS"> + <task xsi:type="execute" hosts="master"> + <script>scripts/namenode.py</script> + <function>restore_snapshot</function> + </task> + </execute-stage> + </group> + + <!-- After processing this group, will change the effective Stack of the UpgradeContext object. --> + <group xsi:type="update-stack" name="UPDATE_DESIRED_STACK_ID" title="Update Desired Stack Id"> + <execute-stage title="Update Desired Stack Id" service="" component=""> + <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.UpdateDesiredStackAction"> + </task> + </execute-stage> + </group> + + <group xsi:type="cluster" name="ALL_HOST_OPS" title="Set Version On All Hosts"> + <skippable>true</skippable> + <execute-stage title="Update stack to {{version}}"> + <task xsi:type="execute"> + <script>scripts/ru_set_all.py</script> + <function>actionexecute</function> + </task> + </execute-stage> + </group> + + <!-- Now, restart all of the services. --> + + <group xsi:type="restart" name="ZOOKEEPER" title="Zookeeper"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="ZOOKEEPER"> + <service-check>false</service-check> + <component>ZOOKEEPER_SERVER</component> + <component>ZOOKEEPER_CLIENT</component> + </service> + </group> + + <group xsi:type="restart" name="HDFS" title="HDFS"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="HDFS"> + <component>JOURNALNODE</component> + <component>ZKFC</component> + <component>NAMENODE</component> + <component>SECONDARY_NAMENODE</component> + <component>DATANODE</component> + <component>HDFS_CLIENT</component> + </service> + </group> + + <group xsi:type="restart" name="MR and YARN" title="MR and YARN"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="MAPREDUCE2"> + <component>HISTORYSERVER</component> + <component>MAPREDUCE2_CLIENT</component> + </service> + <service name="YARN"> + <component>APP_TIMELINE_SERVER</component> + <component>RESOURCEMANAGER</component> + <component>NODEMANAGER</component> + <component>YARN_CLIENT</component> + </service> + </group> + + <group xsi:type="restart" name="HBASE" title="HBASE"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="HBASE"> + <component>HBASE_MASTER</component> + <component>HBASE_REGIONSERVER</component> + <component>HBASE_CLIENT</component> + </service> + </group> + + <group xsi:type="restart" name="CLIENTS" title="Tez, Pig, Sqoop Clients"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="TEZ"> + <component>TEZ_CLIENT</component> + </service> + + <service name="PIG"> + <component>PIG</component> + </service> + + <service name="SQOOP"> + <component>SQOOP</component> + </service> + </group> + + <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check"> + <skippable>true</skippable> + <direction>UPGRADE</direction> + <priority> + <service>HBASE</service> + <service>MAPREDUCE2</service> + <service>YARN</service> + <service>HDFS</service> + </priority> + </group> + + <group xsi:type="restart" name="HIVE" title="Hive"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="HIVE"> + <component>HIVE_METASTORE</component> + <component>HIVE_SERVER</component> + <component>WEBHCAT_SERVER</component> + <component>HIVE_CLIENT</component> + <component>HCAT</component> + </service> + </group> + + <!-- Upgrade Oozie DB only on Upgrade direction, and always create a new ShareLib. --> + <group name="Upgrade Oozie" title="Upgrade Oozie Database"> + <direction>UPGRADE</direction> + <skippable>true</skippable> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Upgrade Oozie Database"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> + <script>scripts/oozie_server_upgrade.py</script> + <function>upgrade_oozie_database_and_sharelib</function> + </task> + </execute-stage> + </group> + + <!-- Only create the ShareLib folder during a Downgrade. --> + <group name="Downgrade Oozie" title="Downgrade Oozie ShareLib"> + <direction>DOWNGRADE</direction> + <skippable>true</skippable> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Downgrade Oozie ShareLib"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> + <script>scripts/oozie_server_upgrade.py</script> + <function>create_sharelib</function> + </task> + </execute-stage> + </group> + + <group xsi:type="restart" name="OOZIE" title="Oozie"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="OOZIE"> + <component>OOZIE_SERVER</component> + <component>OOZIE_CLIENT</component> + </service> + </group> + + <group xsi:type="restart" name="FALCON" title="Falcon"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="FALCON"> + <component>FALCON_SERVER</component> + <component>FALCON_CLIENT</component> + </service> + </group> + + <group xsi:type="restart" name="STORM" title="Storm"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="STORM"> + <component>NIMBUS</component> + <component>STORM_REST_API</component> + <component>SUPERVISOR</component> + <component>STORM_UI_SERVER</component> + <component>DRPC_SERVER</component> + </service> + + <execute-stage service="STORM" component="DRPC_SERVER" title="Rebuild Storm Topology"> + <task xsi:type="manual"> + <message>Please rebuild your topology using the new Storm version dependencies and resubmit it using the newly created jar.</message> + </task> + </execute-stage> + </group> + + <group xsi:type="restart" name="FLUME" title="Flume"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="FLUME"> + <component>FLUME_HANDLER</component> + </service> + </group> + + <group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}"> + <skippable>true</skippable> + <execute-stage title="Check Unhealthy Hosts" id="unhealthy-hosts"> + <task xsi:type="manual"> + <message>The following hosts were unhealthy and should be resolved before finalizing can be completed: {{hosts.unhealthy}}</message> + </task> + </execute-stage> + + <execute-stage title="Confirm Finalize"> + <direction>UPGRADE</direction> + <task xsi:type="manual"> + <message>Please confirm you are ready to finalize.</message> + </task> + </execute-stage> + + <execute-stage service="HDFS" component="NAMENODE" title="Execute HDFS Finalize"> + <task xsi:type="execute" hosts="master"> + <script>scripts/namenode.py</script> + <function>finalize_non_rolling_upgrade</function> + </task> + </execute-stage> + + <execute-stage title="Save Cluster State" service="" component=""> + <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FinalizeUpgradeAction"> + </task> + </execute-stage> + </group> + </order> +</upgrade> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/c58162fe/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml new file mode 100644 index 0000000..9c96dfb --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml @@ -0,0 +1,55 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> + + <services> + <service name="HIVE"> + <component name="HIVE_SERVER"> + <changes> + <definition xsi:type="configure" id="hdp_2_2_0_0_hive_server_set_transport_mode"> + <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> + <type>hive-site</type> + <key>hive.server2.thrift.port</key> + <value>10010</value> + </condition> + <condition type="hive-site" key="hive.server2.transport.mode" value="http"> + <type>hive-site</type> + <key>hive.server2.http.port</key> + <value>10011</value> + </condition> + </definition> + + <definition xsi:type="configure" id="hdp_2_2_0_0_hive_server_restore_transport_mode_on_downgrade"> + <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> + <type>hive-site</type> + <key>hive.server2.thrift.port</key> + <value>10000</value> + </condition> + <condition type="hive-site" key="hive.server2.transport.mode" value="http"> + <type>hive-site</type> + <key>hive.server2.http.port</key> + <value>10001</value> + </condition> + </definition> + </changes> + </component> + </service> + </services> + +</upgrade-config-changes> http://git-wip-us.apache.org/repos/asf/ambari/blob/c58162fe/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml new file mode 100644 index 0000000..8fbb963 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml @@ -0,0 +1,469 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + + +<upgrade xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> + <target>2.2.*.*</target> + <target-stack>HDP-2.2</target-stack> + <type>NON_ROLLING</type> + <prerequisite-checks> + </prerequisite-checks> + <order> + <group xsi:type="cluster" name="PRE_CLUSTER" title="Prepare Upgrade"> + <skippable>true</skippable> + <direction>UPGRADE</direction> + <execute-stage service="YARN" component="RESOURCEMANAGER" title="Stop YARN Queues"> + <task xsi:type="manual"> + <message>Before continuing, please stop all YARN queues. If yarn-site's yarn.resourcemanager.work-preserving-recovery.enabled is set to true, then you can skip this step since the clients will retry on their own.</message> + </task> + </execute-stage> + + <execute-stage service="SLIDER" component="SLIDER" title="Stop Long Running Applications on Slider"> + <task xsi:type="manual"> + <message>Before continuing, please stop all long-running applications deployed using Slider. E.g., su - yarn "/usr/hdp/current/slider-client/bin/slider stop <app_name>"</message> + </task> + </execute-stage> + + <execute-stage service="STORM" component="NIMBUS" title="Stop Storm Topologies"> + <task xsi:type="manual"> + <message>Before continuing, please deactivate and kill any currently running topologies.</message> + </task> + </execute-stage> + </group> + + <group xsi:type="stop" name="Stop High-Level Daemons" title="Stop Daemons for High-Level Services"> + <skippable>true</skippable> + <service-check>false</service-check> + + <service name="FLUME"> + <component>FLUME_HANDLER</component> + </service> + + <service name="STORM"> + <component>DRPC_SERVER</component> + <component>STORM_UI_SERVER</component> + <component>SUPERVISOR</component> + <component>STORM_REST_API</component> + <component>NIMBUS</component> + </service> + + <service name="KNOX"> + <component>KNOX_GATEWAY</component> + </service> + + <service name="KAFKA"> + <component>KAFKA_BROKER</component> + </service> + + <service name="FALCON"> + <component>FALCON_SERVER</component> + </service> + + <service name="OOZIE"> + <component>OOZIE_SERVER</component> + </service> + + <service name="SPARK"> + <component>SPARK_JOBHISTORYSERVER</component> + </service> + + <service name="HIVE"> + <component>WEBHCAT_SERVER</component> + <component>HIVE_SERVER</component> + <component>HIVE_METASTORE</component> + </service> + + <service name="YARN"> + <component>NODEMANAGER</component> <!-- TODO, parallelize --> + <component>RESOURCEMANAGER</component> + <component>APP_TIMELINE_SERVER</component> + </service> + + <service name="MAPREDUCE2"> + <component>HISTORYSERVER</component> + </service> + </group> + + <group xsi:type="cluster" name="Backups" title="Take Backups"> + <direction>UPGRADE</direction> + <skippable>true</skippable> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Backup Oozie Database"> + <task xsi:type="manual"> + <message>Before continuing, please backup the Oozie Server database on {{oozie-env/oozie_hostname}}.</message> + </task> + </execute-stage> + + <execute-stage service="HIVE" component="HIVE_METASTORE" title="Backup Hive Metastore"> + <task xsi:type="manual"> + <message>Before continuing, please backup the Hive Metastore database located on the following host(s): {{hosts.all}}.</message> + </task> + </execute-stage> + + <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Backup Knox Data"> + <task xsi:type="manual"> + <message>Before continuing, please backup the Knox data. E.g., "cp -RL /etc/knox/data/security ~/knox_backup" on the following host(s): {{hosts.all}}.</message> + </task> + </execute-stage> + + <execute-stage service="HBASE" component="HBASE_MASTER" title="Snapshot HBASE"> + <task xsi:type="execute" hosts="master"> + <script>scripts/hbase_upgrade.py</script> + <function>take_snapshot</function> <!-- TODO, this function used to be called just "snapshot" --> + </task> + </execute-stage> + + <execute-stage service="HDFS" component="NAMENODE" title="Snapshot HDFS"> + <task xsi:type="execute" hosts="master"> <!-- TODO, this can be any NameNode, not just the active. --> + <script>scripts/namenode.py</script> + <function>prepare_non_rolling_upgrade</function> + </task> + </execute-stage> + + <execute-stage service="RANGER" component="RANGER_ADMIN" title="Backup Ranger"> + <task xsi:type="manual"> + <message>Before continuing, please backup the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}.</message> + </task> + </execute-stage> + </group> + + <group xsi:type="stop" name="Stop Low-Level Daemons" title="Stop Daemons for Low-Level Services"> + <skippable>true</skippable> + <service-check>false</service-check> + + <service name="HBASE"> + <component>HBASE_REGIONSERVER</component> + <component>HBASE_MASTER</component> + </service> + + <service name="HDFS"> + <component>DATANODE</component> + <component>NAMENODE</component> + <component>SECONDARY_NAMENODE</component> <!-- TODO, may not be present. --> + <component>ZKFC</component> <!-- TODO, may not be present. --> + <component>JOURNALNODE</component> <!-- TODO, may not be present. --> + </service> + + <service name="RANGER"> + <component>RANGER_USERSYNC</component> + <component>RANGER_ADMIN</component> + </service> + + <service name="ZOOKEEPER"> + <component>ZOOKEEPER_SERVER</component> + </service> + </group> + + <group name="Marker for Downgrade" title="Marker for Downgrade"> + <direction>UPGRADE</direction> + <!-- TODO, if the user attempts a downgrade before this step, they can simply abort. --> + </group> + + <group xsi:type="cluster" name="Restore Backups" title="Restore Backups"> + <direction>DOWNGRADE</direction> + <skippable>true</skippable> + + <!-- If the user attempts a downgrade after this point, they will need to restore backups + before starting any of the services. --> + + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Backup Oozie Database"> + <task xsi:type="manual"> + <message>Before continuing, please restore the Oozie Server database on {{hosts.all}}.</message> + </task> + </execute-stage> + + <execute-stage service="HIVE" component="HIVE_METASTORE" title="Backup Hive Metastore"> + <task xsi:type="manual"> + <message>Before continuing, please restore the Hive Metastore database located on the following host(s): {{hosts.all}}.</message> + </task> + </execute-stage> + + <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Backup Knox Data"> + <task xsi:type="manual"> + <message>Before continuing, please restore the Knox data. E.g., "cp -RL ~/knox_backup/* /etc/knox/data/security/" on the following host(s): {{hosts.all}}.</message> + </task> + </execute-stage> + + <execute-stage service="HBASE" component="HBASE_MASTER" title="Snapshot HBASE"> + <task xsi:type="execute" hosts="master"> + <script>scripts/hbase_upgrade.py</script> + <function>restore_snapshot</function> <!-- TODO, this function name is new. --> + </task> + </execute-stage> + + <execute-stage service="HDFS" component="NAMENODE" title="Snapshot HDFS"> + <task xsi:type="execute" hosts="master"> <!-- TODO, this can be any NameNode, not just the active. --> + <script>scripts/namenode.py</script> + <function>restore_snapshot</function> <!-- TODO, this function doesn't exist yet. --> + </task> + </execute-stage> + + <execute-stage service="RANGER" component="RANGER_ADMIN" title="Backup Ranger"> + <task xsi:type="manual"> + <message>Before continuing, please restore the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}.</message> + </task> + </execute-stage> + </group> + + <!-- After processing this group, will change the effective Stack of the UpgradeContext object. --> + <group xsi:type="update-stack" name="UPDATE_DESIRED_STACK_ID" title="Update Desired Stack Id"> + <execute-stage title="Update Desired Stack Id" service="" component=""> + <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.UpdateDesiredStackAction"> + </task> + </execute-stage> + </group> + + <group xsi:type="cluster" name="ALL_HOST_OPS" title="Set Version On All Hosts"> + <skippable>true</skippable> + <execute-stage title="Update stack to {{version}}"> + <task xsi:type="execute"> + <script>scripts/ru_set_all.py</script> + <function>actionexecute</function> <!-- TODO, parallelize --> + </task> + </execute-stage> + </group> + + <!-- Now, restart all of the services. --> + + <group xsi:type="restart" name="ZOOKEEPER" title="Zookeeper"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="ZOOKEEPER"> + <service-check>false</service-check> <!-- TODO, enable service-check once done testing --> + <component>ZOOKEEPER_SERVER</component> + <component>ZOOKEEPER_CLIENT</component> <!-- TODO, parallelize --> + </service> + </group> + + <group xsi:type="restart" name="RANGER" title="Ranger"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="RANGER"> + <component>RANGER_ADMIN</component> + <component>RANGER_USERSYNC</component> + </service> + </group> + + <group xsi:type="restart" name="HDFS" title="HDFS"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="HDFS"> + <component>JOURNALNODE</component> + <component>ZKFC</component> + <component>NAMENODE</component> + <component>SECONDARY_NAMENODE</component> <!-- TODO, may not be present --> + <component>DATANODE</component> <!-- TODO, parallelize --> + <component>HDFS_CLIENT</component> <!-- TODO, parallelize --> + </service> + </group> + + <group xsi:type="restart" name="MR and YARN" title="MR and YARN"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="MAPREDUCE2"> + <component>HISTORYSERVER</component> + <component>MAPREDUCE2_CLIENT</component> <!-- TODO, parallelize --> + </service> + <service name="YARN"> + <component>APP_TIMELINE_SERVER</component> + <component>RESOURCEMANAGER</component> + <component>NODEMANAGER</component> <!-- TODO, parallelize --> + <component>YARN_CLIENT</component> <!-- TODO, parallelize --> + </service> + </group> + + <group xsi:type="restart" name="HBASE" title="HBASE"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="HBASE"> + <component>HBASE_MASTER</component> + <component>HBASE_REGIONSERVER</component> <!-- TODO, parallelize --> + <component>HBASE_CLIENT</component> <!-- TODO, parallelize --> + </service> + </group> + + <group xsi:type="restart" name="CLIENTS" title="Tez, Pig, Sqoop Clients"> <!-- TODO, parallelize --> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="TEZ"> + <component>TEZ_CLIENT</component> + </service> + + <service name="PIG"> + <component>PIG</component> + </service> + + <service name="SQOOP"> + <component>SQOOP</component> + </service> + </group> + + <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check"> + <skippable>true</skippable> + <direction>UPGRADE</direction> + <priority> + <!-- TODO, for some reason, it flips the order. --> + <service>HBASE</service> + <service>MAPREDUCE2</service> + <service>YARN</service> + <service>HDFS</service> + </priority> + </group> + + <group xsi:type="restart" name="HIVE" title="Hive"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="HIVE"> + <component>HIVE_METASTORE</component> + <component>HIVE_SERVER</component> + <component>WEBHCAT_SERVER</component> + <component>HIVE_CLIENT</component> <!-- TODO, parallelize --> + <component>HCAT</component> <!-- TODO, parallelize --> + </service> + </group> + + <group xsi:type="restart" name="SPARK" title="Spark"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="SPARK"> + <component>SPARK_JOBHISTORYSERVER</component> + <component>SPARK_CLIENT</component> <!-- TODO, parallelize --> + </service> + </group> + + <!-- Upgrade Oozie DB only on Upgrade direction, and always create a new ShareLib. --> + <group name="Upgrade Oozie" title="Upgrade Oozie Database"> + <direction>UPGRADE</direction> + <skippable>true</skippable> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Upgrade Oozie Database"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> + <script>scripts/oozie_server_upgrade.py</script> + <function>upgrade_oozie_database_and_sharelib</function> + </task> + </execute-stage> + </group> + + <!-- Only create the ShareLib folder during a Downgrade. --> + <group name="Downgrade Oozie" title="Downgrade Oozie ShareLib"> + <direction>DOWNGRADE</direction> + <skippable>true</skippable> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Downgrade Oozie ShareLib"> + <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> + <script>scripts/oozie_server_upgrade.py</script> + <function>create_sharelib</function> + </task> + </execute-stage> + </group> + + <group xsi:type="restart" name="OOZIE" title="Oozie"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="OOZIE"> + <component>OOZIE_SERVER</component> + <component>OOZIE_CLIENT</component> <!-- TODO, parallelize --> + </service> + </group> + + <group xsi:type="restart" name="FALCON" title="Falcon"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="FALCON"> + <component>FALCON_SERVER</component> + <component>FALCON_CLIENT</component> <!-- TODO, parallelize --> + </service> + </group> + + <group xsi:type="restart" name="KAFKA" title="Kafka"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="KAFKA"> + <component>KAFKA_BROKER</component> + </service> + </group> + + <group xsi:type="restart" name="KNOX" title="Knox"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="KNOX"> + <component>KNOX_GATEWAY</component> + </service> + </group> + + <group xsi:type="restart" name="STORM" title="Storm"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="STORM"> + <component>NIMBUS</component> + <component>STORM_REST_API</component> + <component>SUPERVISOR</component> + <component>STORM_UI_SERVER</component> + <component>DRPC_SERVER</component> + </service> + + <!-- TODO, does this work? --> + <execute-stage service="STORM" component="DRPC_SERVER" title="Rebuild Storm Topology"> + <task xsi:type="manual"> + <message>Please rebuild your topology using the new Storm version dependencies and resubmit it using the newly created jar.</message> + </task> + </execute-stage> + </group> + + <group xsi:type="restart" name="SLIDER" title="Slider"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="SLIDER"> + <component>SLIDER</component> + </service> + </group> + + <group xsi:type="restart" name="FLUME" title="Flume"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="FLUME"> + <component>FLUME_HANDLER</component> + </service> + </group> + + <group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}"> + <skippable>true</skippable> + <execute-stage title="Check Unhealthy Hosts" id="unhealthy-hosts"> + <task xsi:type="manual"> + <message>The following hosts were unhealthy and should be resolved before finalizing can be completed: {{hosts.unhealthy}}</message> + </task> + </execute-stage> + + <execute-stage title="Confirm Finalize"> + <direction>UPGRADE</direction> + <task xsi:type="manual"> + <message>Please confirm you are ready to finalize.</message> + </task> + </execute-stage> + + <execute-stage service="HDFS" component="NAMENODE" title="Execute HDFS Finalize"> + <task xsi:type="execute" hosts="master"> <!-- TODO, what happens if there's no HA. --> + <script>scripts/namenode.py</script> + <function>finalize_non_rolling_upgrade</function> + </task> + </execute-stage> + + <execute-stage title="Save Cluster State" service="" component=""> + <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FinalizeUpgradeAction"> + </task> + </execute-stage> + </group> + </order> +</upgrade> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/c58162fe/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml index 5920b72..810423d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml @@ -21,7 +21,20 @@ <target>2.2.*.*</target> <skip-failures>false</skip-failures> <skip-service-check-failures>false</skip-service-check-failures> - + <target-stack>HDP-2.3</target-stack> + <type>ROLLING</type> + <prerequisite-checks> + <check>org.apache.ambari.server.checks.HiveMultipleMetastoreCheck</check> + <check>org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck</check> + <check>org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck</check> + <check>org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck</check> + <check>org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck</check> + <check>org.apache.ambari.server.checks.ServicesNamenodeTruncateCheck</check> + <check>org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck</check> + <check>org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck</check> + <check>org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck</check> + <check>org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck</check> + </prerequisite-checks> <order> <group xsi:type="cluster" name="PRE_CLUSTER" title="Prepare Upgrade"> <direction>UPGRADE</direction> @@ -468,18 +481,7 @@ <message>Please note that the HiveServer port will now change to 10010 if hive is using a binary transfer mode or 10011 if hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> </task> - <task xsi:type="configure"> - <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> - <type>hive-site</type> - <key>hive.server2.thrift.port</key> - <value>10010</value> - </condition> - <condition type="hive-site" key="hive.server2.transport.mode" value="http"> - <type>hive-site</type> - <key>hive.server2.http.port</key> - <value>10011</value> - </condition> - </task> + <task xsi:type="configure" id="hdp_2_2_0_0_hive_server_set_transport_mode"/> </pre-upgrade> <pre-downgrade> @@ -488,18 +490,7 @@ <message>Please note that the HiveServer port will now change to 10000 if hive is using a binary transfer mode or 10001 if hive is using an http transport mode. You can use "netstat -anp | grep 1000[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> </task> - <task xsi:type="configure"> - <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> - <type>hive-site</type> - <key>hive.server2.thrift.port</key> - <value>10000</value> - </condition> - <condition type="hive-site" key="hive.server2.transport.mode" value="http"> - <type>hive-site</type> - <key>hive.server2.http.port</key> - <value>10001</value> - </condition> - </task> + <task xsi:type="configure" id="hdp_2_2_0_0_hive_server_restore_transport_mode_on_downgrade"/> </pre-downgrade> <upgrade>
