AMBARI-20522. Upgrade support for druid components (Nishant Bangarwa via ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e0be5f6d Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e0be5f6d Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e0be5f6d Branch: refs/heads/branch-feature-AMBARI-12556 Commit: e0be5f6d4520c76d14ed5883fc402ec380e1bb11 Parents: fbb0af3 Author: Nate Cole <[email protected]> Authored: Wed Mar 22 16:50:45 2017 -0400 Committer: Nate Cole <[email protected]> Committed: Wed Mar 22 16:50:45 2017 -0400 ---------------------------------------------------------------------- .../common-services/DRUID/0.9.2/metainfo.xml | 7 +++ .../DRUID/0.9.2/package/scripts/druid_node.py | 16 ++++- .../DRUID/0.9.2/package/scripts/params.py | 2 + .../DRUID/0.9.2/package/scripts/superset.py | 42 +++++++++---- .../HDP/2.6/upgrades/host-upgrade-2.6.xml | 45 ++++++++++++++ .../HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml | 64 ++++++++++++++++++++ .../stacks/HDP/2.6/upgrades/upgrade-2.6.xml | 54 +++++++++++++++++ 7 files changed, 215 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/e0be5f6d/ambari-server/src/main/resources/common-services/DRUID/0.9.2/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/metainfo.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/metainfo.xml index c0859f2..404545c 100644 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/metainfo.xml @@ -29,6 +29,7 @@ <displayName>Druid Coordinator</displayName> <category>MASTER</category> <cardinality>1+</cardinality> + <versionAdvertised>true</versionAdvertised> <dependencies> <dependency> <name>HDFS/HDFS_CLIENT</name> @@ -52,6 +53,7 @@ <displayName>Druid Overlord</displayName> <category>MASTER</category> <cardinality>1+</cardinality> + <versionAdvertised>true</versionAdvertised> <dependencies> <dependency> <name>HDFS/HDFS_CLIENT</name> @@ -75,6 +77,7 @@ <displayName>Druid Historical</displayName> <category>SLAVE</category> <cardinality>1+</cardinality> + <versionAdvertised>true</versionAdvertised> <dependencies> <dependency> <name>HDFS/HDFS_CLIENT</name> @@ -98,6 +101,7 @@ <displayName>Druid Broker</displayName> <category>MASTER</category> <cardinality>1+</cardinality> + <versionAdvertised>true</versionAdvertised> <dependencies> <dependency> <name>HDFS/HDFS_CLIENT</name> @@ -121,6 +125,7 @@ <displayName>Druid MiddleManager</displayName> <category>SLAVE</category> <cardinality>1+</cardinality> + <versionAdvertised>true</versionAdvertised> <dependencies> <dependency> <name>HDFS/HDFS_CLIENT</name> @@ -144,6 +149,7 @@ <displayName>Druid Router</displayName> <category>MASTER</category> <cardinality>1+</cardinality> + <versionAdvertised>true</versionAdvertised> <dependencies> <dependency> <name>HDFS/HDFS_CLIENT</name> @@ -167,6 +173,7 @@ <displayName>Druid Superset</displayName> <category>MASTER</category> <cardinality>1+</cardinality> + <versionAdvertised>true</versionAdvertised> <commandScript> <script>scripts/superset.py</script> <scriptType>PYTHON</scriptType> http://git-wip-us.apache.org/repos/asf/ambari/blob/e0be5f6d/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid_node.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid_node.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid_node.py index 6c1cc20..7c6bf39 100644 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid_node.py +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid_node.py @@ -22,8 +22,11 @@ from resource_management import Script from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute from resource_management.libraries.functions.format import format +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions.check_process_status import check_process_status - from resource_management.libraries.functions.show_logs import show_logs from druid import druid, get_daemon_cmd, getPid @@ -45,7 +48,16 @@ class DruidBase(Script): druid(upgrade_type=upgrade_type, nodeType=self.nodeType) def pre_upgrade_restart(self, env, upgrade_type=None): - return + node_type_lower = self.nodeType.lower() + Logger.info(format("Executing druid-{node_type_lower} Upgrade pre-restart")) + import params + + env.set_params(params) + + if params.stack_version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.stack_version): + stack_select.select(self.get_component_name(), params.stack_version) + if params.stack_version and check_stack_feature(StackFeature.CONFIG_VERSIONING, params.stack_version): + conf_select.select(params.stack_name, "druid", params.stack_version) def start(self, env, upgrade_type=None): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/e0be5f6d/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py index 08ff3ab..b1d9472 100644 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py @@ -45,6 +45,8 @@ config = Script.get_config() stack_root = Script.get_stack_root() tmp_dir = Script.get_tmp_dir() +stack_name = default("/hostLevelParams/stack_name", None) + # stack version stack_version = default("/commandParams/version", None) http://git-wip-us.apache.org/repos/asf/ambari/blob/e0be5f6d/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/superset.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/superset.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/superset.py index 9173b09..f835418 100644 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/superset.py +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/superset.py @@ -16,17 +16,23 @@ See the License for the specific language governing permissions and limitations under the License. """ -import os, time -from resource_management.core.resources.system import Directory +import os + from resource_management import Script -from resource_management.libraries.resources.properties_file import PropertiesFile +from resource_management.core.logger import Logger +from resource_management.core.resources import File +from resource_management.core.resources.system import Directory from resource_management.core.resources.system import Execute -from resource_management.core.source import Template -from resource_management.libraries.functions.show_logs import show_logs from resource_management.core.source import InlineTemplate -from resource_management.libraries.functions.format import format +from resource_management.core.source import Template +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select from resource_management.libraries.functions.check_process_status import check_process_status -from resource_management.core.resources import File +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.show_logs import show_logs +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.resources.properties_file import PropertiesFile class Superset(Script): @@ -36,7 +42,7 @@ class Superset(Script): def install(self, env): self.install_packages(env) - def configure(self, env): + def configure(self, env, upgrade_type=None): import params Directory( [params.superset_pid_dir, params.superset_log_dir, params.superset_config_dir, params.superset_home_dir], @@ -85,10 +91,21 @@ class Superset(Script): Execute(format("source {params.superset_config_dir}/superset-env.sh ; {params.superset_bin_dir}/superset configure_druid_cluster --name druid-ambari --coordinator-host {params.druid_coordinator_host} --coordinator-port {params.druid_coordinator_port} --broker-host {params.druid_router_host} --broker-port {params.druid_router_port} --coordinator-endpoint druid/coordinator/v1/metadata --broker-endpoint druid/v2"), user=params.druid_user) - def start(self, env): + def pre_upgrade_restart(self, env, upgrade_type=None): + Logger.info("Executing druid-superset Upgrade pre-restart") + import params + + env.set_params(params) + + if params.stack_version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.stack_version): + stack_select.select(self.get_component_name(), params.stack_version) + if params.stack_version and check_stack_feature(StackFeature.CONFIG_VERSIONING, params.stack_version): + conf_select.select(params.stack_name, "superset", params.stack_version) + + def start(self, env, upgrade_type=None): import params env.set_params(params) - self.configure(env) + self.configure(env, upgrade_type=upgrade_type) daemon_cmd = self.get_daemon_cmd(params, "start") try: Execute(daemon_cmd, @@ -98,11 +115,10 @@ class Superset(Script): show_logs(params.druid_log_dir, params.druid_user) raise - - def stop(self, env): + def stop(self, env, upgrade_type=None): import params env.set_params(params) - + self.configure(env, upgrade_type=upgrade_type) daemon_cmd = self.get_daemon_cmd(params, "stop") try: Execute(daemon_cmd, http://git-wip-us.apache.org/repos/asf/ambari/blob/e0be5f6d/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/host-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/host-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/host-upgrade-2.6.xml index 087482c..f923702 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/host-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/host-upgrade-2.6.xml @@ -558,5 +558,50 @@ </upgrade> </component> </service> + + <service name="DRUID"> + <component name="DRUID_HISTORICAL"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + + <component name="DRUID_MIDDLEMANAGER"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + + <component name="DRUID_BROKER"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + + <component name="DRUID_ROUTER"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + + <component name="DRUID_COORDINATOR"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + + <component name="DRUID_OVERLORD"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + + <component name="DRUID_SUPERSET"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + </service> + </processing> </upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/e0be5f6d/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml index 035cb59..4a641ec 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml @@ -135,6 +135,17 @@ <service name="MAPREDUCE2"> <component>HISTORYSERVER</component> </service> + + <service name="DRUID"> + <component>DRUID_HISTORICAL</component> + <component>DRUID_MIDDLEMANAGER</component> + <component>DRUID_BROKER</component> + <component>DRUID_ROUTER</component> + <component>DRUID_COORDINATOR</component> + <component>DRUID_OVERLORD</component> + <component>DRUID_SUPERSET</component> + </service> + </group> <group xsi:type="cluster" name="Backups" title="Perform Backups"> @@ -687,6 +698,19 @@ </service> </group> + <group xsi:type="restart" name="DRUID" title="Druid"> + <skippable>true</skippable> + <service name="DRUID"> + <component>DRUID_HISTORICAL</component> + <component>DRUID_MIDDLEMANAGER</component> + <component>DRUID_BROKER</component> + <component>DRUID_ROUTER</component> + <component>DRUID_COORDINATOR</component> + <component>DRUID_OVERLORD</component> + <component>DRUID_SUPERSET</component> + </service> + </group> + <group name="SERVICE_CHECK_3" title="All Service Checks" xsi:type="service-check"> <direction>UPGRADE</direction> <skippable>true</skippable> @@ -696,6 +720,7 @@ <service>STORM</service> <service>FLUME</service> <service>ACCUMULO</service> + <service>DRUID</service> </priority> </group> @@ -1200,5 +1225,44 @@ </upgrade> </component> </service> + + <service name="DRUID"> + <component name="DRUID_HISTORICAL"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + <component name="DRUID_MIDDLEMANAGER"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + <component name="DRUID_BROKER"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + <component name="DRUID_ROUTER"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + <component name="DRUID_COORDINATOR"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + <component name="DRUID_OVERLORD"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + <component name="DRUID_SUPERSET"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + </service> + </processing> </upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/e0be5f6d/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml index 458667d..0587741 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml @@ -381,6 +381,21 @@ </service> </group> + <group name="DRUID" title="Druid"> + <skippable>true</skippable> + <supports-auto-skip-failure>false</supports-auto-skip-failure> + <service-check>false</service-check> + <service name="DRUID"> + <component>DRUID_HISTORICAL</component> + <component>DRUID_MIDDLEMANAGER</component> + <component>DRUID_BROKER</component> + <component>DRUID_ROUTER</component> + <component>DRUID_COORDINATOR</component> + <component>DRUID_OVERLORD</component> + <component>DRUID_SUPERSET</component> + </service> + </group> + <group name="SERVICE_CHECK_3" title="All Service Checks" xsi:type="service-check"> <direction>UPGRADE</direction> <skippable>true</skippable> @@ -389,6 +404,7 @@ <service>HDFS</service> <service>YARN</service> <service>HBASE</service> + <service>DRUID</service> </priority> <exclude> <service>AMBARI_METRICS</service> @@ -971,5 +987,43 @@ </component> </service> + <service name="DRUID"> + <component name="DRUID_HISTORICAL"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + <component name="DRUID_MIDDLEMANAGER"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + <component name="DRUID_BROKER"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + <component name="DRUID_ROUTER"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + <component name="DRUID_COORDINATOR"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + <component name="DRUID_OVERLORD"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + <component name="DRUID_SUPERSET"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + </service> + </processing> </upgrade>
