Repository: ambari Updated Branches: refs/heads/trunk 381ace20d -> 89fd30b40
AMBARI-15939 - Atlas Integration :Use stack advisor to suggest correct integration related configs Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/89fd30b4 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/89fd30b4 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/89fd30b4 Branch: refs/heads/trunk Commit: 89fd30b40f3108bfcbcc73eb2d74c94a2ba14a7a Parents: 381ace2 Author: tbeerbower <[email protected]> Authored: Tue Apr 19 09:21:40 2016 -0400 Committer: tbeerbower <[email protected]> Committed: Tue Apr 19 09:21:40 2016 -0400 ---------------------------------------------------------------------- .../FALCON/0.5.0.2.1/package/scripts/falcon.py | 19 +- .../0.5.0.2.1/package/scripts/params_linux.py | 25 +- .../package/scripts/setup_atlas_falcon.py | 52 ++++ .../1.4.4.2.0/configuration/sqoop-site.xml | 7 - .../1.4.4.2.0/package/scripts/params_linux.py | 6 + .../package/scripts/setup_atlas_sqoop.py | 47 ++++ .../SQOOP/1.4.4.2.0/package/scripts/sqoop.py | 16 +- .../0.9.1.2.1/configuration/storm-site.xml | 7 - .../0.9.1.2.1/package/scripts/params_linux.py | 6 + .../package/scripts/setup_atlas_storm.py | 45 ++++ .../STORM/0.9.1.2.1/package/scripts/storm.py | 13 +- .../configuration/falcon-startup.properties.xml | 10 +- .../stacks/HDP/2.3/services/stack_advisor.py | 70 ++++- .../stacks/2.3/common/test_stack_advisor.py | 258 +++++++++++++++++++ 14 files changed, 505 insertions(+), 76 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py index 46457ad..6e2e678 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py @@ -27,7 +27,6 @@ from resource_management.core.resources.service import Service from resource_management.core.resources.service import ServiceConfig from resource_management.core.resources.system import Directory from resource_management.core.resources.system import File -from resource_management.core.resources.system import Link from resource_management.libraries.script import Script from resource_management.libraries.resources import PropertiesFile from resource_management.libraries.functions import format @@ -35,6 +34,7 @@ from resource_management.libraries.functions.show_logs import show_logs from ambari_commons import OSConst from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from setup_atlas_falcon import setup_atlas_falcon @OsFamilyFuncImpl(os_family = OsFamilyImpl.DEFAULT) def falcon(type, action = None, upgrade_type=None): @@ -108,12 +108,7 @@ def falcon(type, action = None, upgrade_type=None): create_parents = True, cd_access = "a") - if params.has_atlas: - atlas_falcon_hook_dir = os.path.join(params.atlas_home_dir, "hook", "falcon") - if os.path.exists(atlas_falcon_hook_dir): - Link(os.path.join(params.falcon_conf_dir, params.atlas_conf_file), - to = os.path.join(params.atlas_conf_dir, params.atlas_conf_file) - ) + setup_atlas_falcon() if type == 'server': if action == 'config': @@ -189,16 +184,6 @@ def falcon(type, action = None, upgrade_type=None): show_logs(params.falcon_log_dir, params.falcon_user) raise - if params.has_atlas: - atlas_falcon_hook_dir = os.path.join(params.atlas_home_dir, "hook", "falcon") - if os.path.exists(atlas_falcon_hook_dir): - src_files = os.listdir(atlas_falcon_hook_dir) - for file_name in src_files: - atlas_falcon_hook_file_name = os.path.join(atlas_falcon_hook_dir, file_name) - falcon_lib_file_name = os.path.join(params.falcon_webinf_lib, file_name) - if (os.path.isfile(atlas_falcon_hook_file_name)): - Link(falcon_lib_file_name, to = atlas_falcon_hook_file_name) - if action == 'stop': try: Execute(format('{falcon_home}/bin/falcon-stop'), http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py index 84d8b57..77e18ff 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py @@ -26,6 +26,7 @@ from resource_management.libraries.functions.get_not_managed_resources import ge from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.script.script import Script import os +from resource_management.libraries.functions.expect import expect from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions import StackFeature @@ -33,6 +34,9 @@ config = Script.get_config() stack_root = status_params.stack_root stack_name = default("/hostLevelParams/stack_name", None) +agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_retry_on_unavailability'] +agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int) + # New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade version = default("/commandParams/version", None) @@ -112,31 +116,14 @@ dfs_data_mirroring_dir = "/apps/data-mirroring" atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', []) has_atlas = len(atlas_hosts) > 0 +atlas_plugin_package = "atlas-metadata*-hive-plugin" +atlas_ubuntu_plugin_package = "atlas-metadata.*-hive-plugin" if has_atlas: atlas_conf_file = config['configurations']['atlas-env']['metadata_conf_file'] atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf' atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else format('{stack_root}/current/atlas-server') - application_services = "org.apache.falcon.security.AuthenticationInitializationService,\ - org.apache.falcon.workflow.WorkflowJobEndNotificationService, \ - org.apache.falcon.service.ProcessSubscriberService,\ - org.apache.falcon.entity.store.ConfigurationStore,\ - org.apache.falcon.rerun.service.RetryService,\ - org.apache.falcon.rerun.service.LateRunService,\ - org.apache.falcon.service.LogCleanupService,\ - org.apache.falcon.metadata.MetadataMappingService,\ - org.apache.falcon.atlas.service.AtlasService" -else: - application_services = "org.apache.falcon.security.AuthenticationInitializationService,\ - org.apache.falcon.workflow.WorkflowJobEndNotificationService, \ - org.apache.falcon.service.ProcessSubscriberService,\ - org.apache.falcon.entity.store.ConfigurationStore,\ - org.apache.falcon.rerun.service.RetryService,\ - org.apache.falcon.rerun.service.LateRunService,\ - org.apache.falcon.service.LogCleanupService,\ - org.apache.falcon.metadata.MetadataMappingService" - hdfs_site = config['configurations']['hdfs-site'] default_fs = config['configurations']['core-site']['fs.defaultFS'] http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/setup_atlas_falcon.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/setup_atlas_falcon.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/setup_atlas_falcon.py new file mode 100644 index 0000000..4b7c0c3 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/setup_atlas_falcon.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.resources.packaging import Package +from resource_management.core.resources.system import Directory +from resource_management.core.resources.system import Link +from ambari_commons import OSCheck + +import os + +def setup_atlas_falcon(): + import params + + if params.has_atlas: + + if not params.host_sys_prepped: + Package(params.atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else params.atlas_plugin_package, + retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count) + + atlas_falcon_hook_dir = os.path.join(params.atlas_home_dir, "hook", "falcon") + if os.path.exists(atlas_falcon_hook_dir): + Link(os.path.join(params.falcon_conf_dir, params.atlas_conf_file), + to = os.path.join(params.atlas_conf_dir, params.atlas_conf_file) + ) + + Directory(params.falcon_webinf_lib, + owner = params.falcon_user, + create_parents = True) + + src_files = os.listdir(atlas_falcon_hook_dir) + for file_name in src_files: + atlas_falcon_hook_file_name = os.path.join(atlas_falcon_hook_dir, file_name) + falcon_lib_file_name = os.path.join(params.falcon_webinf_lib, file_name) + if (os.path.isfile(atlas_falcon_hook_file_name)): + Link(falcon_lib_file_name, to = atlas_falcon_hook_file_name) http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml index dd299fe..1bc4c43 100644 --- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml @@ -32,12 +32,5 @@ </property> <property> <name>sqoop.job.data.publish.class</name> - <value>{{job_data_publish_class}}</value> - <depends-on> - <property> - <type>application-properties</type> - <name>atlas.cluster.name</name> - </property> - </depends-on> </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py index 7db974f..658e69f 100644 --- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py @@ -24,6 +24,7 @@ from resource_management.libraries.script import Script from resource_management.libraries.functions.format import format from resource_management.libraries.functions import StackFeature from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.expect import expect import os # a map of the Ambari role to the component name @@ -46,6 +47,9 @@ stack_name = default("/hostLevelParams/stack_name", None) stack_version_unformatted = config['hostLevelParams']['stack_version'] stack_version_formatted = format_stack_version(stack_version_unformatted) +agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_retry_on_unavailability'] +agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int) + # New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade version = default("/commandParams/version", None) @@ -116,6 +120,8 @@ job_data_publish_class = '' atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', []) has_atlas = len(atlas_hosts) > 0 +atlas_plugin_package = "atlas-metadata*-hive-plugin" +atlas_ubuntu_plugin_package = "atlas-metadata.*-hive-plugin" if has_atlas: atlas_conf_file = config['configurations']['atlas-env']['metadata_conf_file'] http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/setup_atlas_sqoop.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/setup_atlas_sqoop.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/setup_atlas_sqoop.py new file mode 100644 index 0000000..d18d820 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/setup_atlas_sqoop.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.resources.packaging import Package +from resource_management.core.resources.system import Link +from ambari_commons import OSCheck + +import os + +def setup_atlas_sqoop(): + import params + + if params.has_atlas: + + if not params.host_sys_prepped: + Package(params.atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else params.atlas_plugin_package, + retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count) + + atlas_sqoop_hook_dir = os.path.join(params.atlas_home_dir, "hook", "sqoop") + if os.path.exists(atlas_sqoop_hook_dir): + Link(os.path.join(params.sqoop_conf_dir, params.atlas_conf_file), + to = os.path.join(params.atlas_conf_dir, params.atlas_conf_file) + ) + + src_files = os.listdir(atlas_sqoop_hook_dir) + for file_name in src_files: + atlas_sqoop_hook_file_name = os.path.join(atlas_sqoop_hook_dir, file_name) + sqoop_lib_file_name = os.path.join(params.sqoop_lib, file_name) + if (os.path.isfile(atlas_sqoop_hook_file_name)): + Link(sqoop_lib_file_name, to = atlas_sqoop_hook_file_name) http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py index 1276a54..799447b 100644 --- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py @@ -23,6 +23,8 @@ from resource_management.libraries.resources.xml_config import XmlConfig from resource_management.core.resources.system import File, Link, Directory from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from ambari_commons import OSConst +from setup_atlas_sqoop import setup_atlas_sqoop + import os @@ -59,19 +61,7 @@ def sqoop(type=None): group = params.user_group ) - if params.has_atlas: - atlas_sqoop_hook_dir = os.path.join(params.atlas_home_dir, "hook", "sqoop") - if os.path.exists(atlas_sqoop_hook_dir): - Link(os.path.join(params.sqoop_conf_dir, params.atlas_conf_file), - to = os.path.join(params.atlas_conf_dir, params.atlas_conf_file) - ) - - src_files = os.listdir(atlas_sqoop_hook_dir) - for file_name in src_files: - atlas_sqoop_hook_file_name = os.path.join(atlas_sqoop_hook_dir, file_name) - sqoop_lib_file_name = os.path.join(params.sqoop_lib, file_name) - if (os.path.isfile(atlas_sqoop_hook_file_name)): - Link(sqoop_lib_file_name, to = atlas_sqoop_hook_file_name) + setup_atlas_sqoop() File(format("{sqoop_conf_dir}/sqoop-env.sh"), owner=params.sqoop_user, http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml index cf702c0..5a3901a 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml @@ -739,12 +739,5 @@ </property> <property> <name>storm.topology.submission.notifier.plugin.class</name> - <value>org.apache.atlas.storm.hook.StormAtlasHook</value> - <depends-on> - <property> - <type>application-properties</type> - <name>atlas.cluster.name</name> - </property> - </depends-on> </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py index 3b65ea9..0752b30 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py @@ -37,6 +37,7 @@ from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.expect import expect # server configurations config = Script.get_config() @@ -50,6 +51,9 @@ stack_name = default("/hostLevelParams/stack_name", None) upgrade_direction = default("/commandParams/upgrade_direction", Direction.UPGRADE) version = default("/commandParams/version", None) +agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_retry_on_unavailability'] +agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int) + storm_component_home_dir = status_params.storm_component_home_dir conf_dir = status_params.conf_dir @@ -190,6 +194,8 @@ jar_jvm_opts = '' # Atlas related params atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', []) has_atlas = len(atlas_hosts) > 0 +atlas_plugin_package = "atlas-metadata*-hive-plugin" +atlas_ubuntu_plugin_package = "atlas-metadata.*-hive-plugin" if has_atlas: atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else stack_root + '/current/atlas-server' http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_atlas_storm.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_atlas_storm.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_atlas_storm.py new file mode 100644 index 0000000..6c3e91f --- /dev/null +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_atlas_storm.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.resources.packaging import Package +from resource_management.core.resources.system import Link +from ambari_commons import OSCheck + +import os + +def setup_atlas_storm(): + import params + + if params.has_atlas: + + if not params.host_sys_prepped: + Package(params.atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else params.atlas_plugin_package, + retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count) + + atlas_storm_hook_dir = os.path.join(params.atlas_home_dir, "hook", "storm") + if os.path.exists(atlas_storm_hook_dir): + storm_extlib_dir = os.path.join(params.storm_component_home_dir, "extlib") + if os.path.exists(storm_extlib_dir): + src_files = os.listdir(atlas_storm_hook_dir) + for file_name in src_files: + atlas_storm_hook_file_name = os.path.join(atlas_storm_hook_dir, file_name) + storm_lib_file_name = os.path.join(storm_extlib_dir, file_name) + if (os.path.isfile(atlas_storm_hook_file_name)): + Link(storm_lib_file_name, to = atlas_storm_hook_file_name) http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py index 74050d8..2d50767 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py @@ -31,6 +31,7 @@ from resource_management.libraries.functions import StackFeature from storm_yaml_utils import yaml_config_template, yaml_config from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from ambari_commons import OSConst +from setup_atlas_storm import setup_atlas_storm @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) def storm(name=None): @@ -94,17 +95,7 @@ def storm(name=None): content=InlineTemplate(params.storm_env_sh_template) ) - if params.has_atlas: - atlas_storm_hook_dir = os.path.join(params.atlas_home_dir, "hook", "storm") - if os.path.exists(atlas_storm_hook_dir): - storm_extlib_dir = os.path.join(params.storm_component_home_dir, "extlib") - if os.path.exists(storm_extlib_dir): - src_files = os.listdir(atlas_storm_hook_dir) - for file_name in src_files: - atlas_storm_hook_file_name = os.path.join(atlas_storm_hook_dir, file_name) - storm_lib_file_name = os.path.join(storm_extlib_dir, file_name) - if (os.path.isfile(atlas_storm_hook_file_name)): - Link(storm_lib_file_name, to = atlas_storm_hook_file_name) + setup_atlas_storm() if params.has_metric_collector: File(format("{conf_dir}/storm-metrics2.properties"), http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml index 5d4fa1e..4e4ec67 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml @@ -27,7 +27,15 @@ </property> <property> <name>*.application.services</name> - <value>{{application_services}}</value> + <value>org.apache.falcon.security.AuthenticationInitializationService,\ + org.apache.falcon.workflow.WorkflowJobEndNotificationService, \ + org.apache.falcon.service.ProcessSubscriberService,\ + org.apache.falcon.entity.store.ConfigurationStore,\ + org.apache.falcon.rerun.service.RetryService,\ + org.apache.falcon.rerun.service.LateRunService,\ + org.apache.falcon.service.LogCleanupService,\ + org.apache.falcon.metadata.MetadataMappingService + </value> <description>Falcon Services</description> </property> <property> http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index 66076f2..e880377 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -161,7 +161,10 @@ class HDP23StackAdvisor(HDP22StackAdvisor): "KAFKA": self.recommendKAFKAConfigurations, "RANGER": self.recommendRangerConfigurations, "RANGER_KMS": self.recommendRangerKMSConfigurations, - "HAWQ": self.recommendHAWQConfigurations + "HAWQ": self.recommendHAWQConfigurations, + "FALCON": self.recommendFalconConfigurations, + "STORM": self.recommendStormConfigurations, + "SQOOP": self.recommendSqoopConfigurations } parentRecommendConfDict.update(childRecommendConfDict) return parentRecommendConfDict @@ -730,6 +733,71 @@ class HDP23StackAdvisor(HDP22StackAdvisor): putHdfsClientProperty = self.putProperty(configurations, "hdfs-client", services) putHdfsClientProperty("output.replace-datanode-on-failure", propertyValue) + def recommendSqoopConfigurations(self, configurations, clusterData, services, hosts): + putSqoopSiteProperty = self.putProperty(configurations, "sqoop-site", services) + + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + if "ATLAS" in servicesList: + putSqoopSiteProperty('sqoop.job.data.publish.class', 'org.apache.atlas.sqoop.hook.SqoopHook') + + def recommendStormConfigurations(self, configurations, clusterData, services, hosts): + putStormStartupProperty = self.putProperty(configurations, "storm-site", services) + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + + # atlas + notifier_plugin_property = "storm.topology.submission.notifier.plugin.class" + if notifier_plugin_property in services["configurations"]["storm-site"]["properties"]: + notifier_plugin_value = services["configurations"]["storm-site"]["properties"][notifier_plugin_property] + else: + notifier_plugin_value = " " + + include_atlas = "ATLAS" in servicesList + atlas_hook_class = "org.apache.atlas.storm.hook.StormAtlasHook" + if include_atlas and atlas_hook_class not in notifier_plugin_value: + if notifier_plugin_value == " ": + notifier_plugin_value = atlas_hook_class + else: + notifier_plugin_value = notifier_plugin_value + "," + atlas_hook_class + if not include_atlas and atlas_hook_class in notifier_plugin_value: + application_classes = [] + for application_class in notifier_plugin_value.split(","): + if application_class != atlas_hook_class and application_class != " ": + application_classes.append(application_class) + if application_classes: + notifier_plugin_value = ",".join(application_classes) + else: + notifier_plugin_value = " " + putStormStartupProperty(notifier_plugin_property, notifier_plugin_value) + + def recommendFalconConfigurations(self, configurations, clusterData, services, hosts): + + putFalconStartupProperty = self.putProperty(configurations, "falcon-startup.properties", services) + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + + # atlas + application_services_property = "*.application.services" + if application_services_property in services["configurations"]["falcon-startup.properties"]["properties"]: + application_services_value = services["configurations"]["falcon-startup.properties"]["properties"][application_services_property] + else: + application_services_value = " " + + include_atlas = "ATLAS" in servicesList + atlas_application_class = "org.apache.falcon.atlas.service.AtlasService" + if include_atlas and atlas_application_class not in application_services_value: + if application_services_value == " ": + application_services_value = atlas_application_class + else: + application_services_value = application_services_value + "," + atlas_application_class + if not include_atlas and atlas_application_class in application_services_value: + application_classes = [] + for application_class in application_services_value.split(","): + if application_class != atlas_application_class and application_class != " ": + application_classes.append(application_class) + if application_classes: + application_services_value = ",".join(application_classes) + else: + application_services_value = " " + putFalconStartupProperty(application_services_property, application_services_value) def getServiceConfigurationValidators(self): parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators() http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 6c9fd46..617f73d 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -2355,3 +2355,261 @@ class TestHDP23StackAdvisor(TestCase): # kms_user was changed, old property should be removed self.stackAdvisor.recommendRangerKMSConfigurations(recommendedConfigurations, clusterData, services, None) self.assertEquals(recommendedConfigurations, expected) + + def test_recommendStormConfigurations(self): + self.maxDiff = None + configurations = { + "storm-site": { + "properties": { + "storm.topology.submission.notifier.plugin.class": "foo" + } + } + } + clusterData = { + "cpu": 4, + "mapMemory": 3000, + "amMemory": 2000, + "reduceMemory": 2056, + "containers": 3, + "ramPerContainer": 256 + } + expected = { + 'storm-site': { + 'properties': { + 'storm.topology.submission.notifier.plugin.class': 'foo,org.apache.atlas.storm.hook.StormAtlasHook', + } + } + } + services = { + "services": [ + { + "href": "/api/v1/stacks/HDP/versions/2.2/services/ATLAS", + "StackServices": { + "service_name": "ATLAS", + "service_version": "2.6.0.2.2", + "stack_name": "HDP", + "stack_version": "2.3" + }, + "components": [ + { + "StackServiceComponents": { + "advertise_version": "false", + "cardinality": "1", + "component_category": "MASTER", + "component_name": "ATLAS_SERVER", + "display_name": "Atlas Server", + "is_client": "false", + "is_master": "true", + "hostnames": [] + }, + "dependencies": [] + } + ] + }, + ], + "configurations": { + "storm-site": { + "properties": { + "storm.topology.submission.notifier.plugin.class": "foo" + } + } + }, + "changed-configurations": [ ] + + } + hosts = { + "items" : [ + { + "href" : "/api/v1/hosts/c6401.ambari.apache.org", + "Hosts" : { + "cpu_count" : 1, + "host_name" : "c6401.ambari.apache.org", + "os_arch" : "x86_64", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "c6401.ambari.apache.org", + "rack_info" : "/default-rack", + "total_mem" : 1922680 + } + } + ] + } + + self.stackAdvisor.recommendStormConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations, expected) + + services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.7.3_23'} + self.stackAdvisor.recommendStormConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations, expected) + + def test_recommendFalconConfigurations(self): + self.maxDiff = None + configurations = { + "falcon-startup.properties": { + "properties": { + "*.application.services": "foo" + } + } + } + clusterData = { + "cpu": 4, + "mapMemory": 3000, + "amMemory": 2000, + "reduceMemory": 2056, + "containers": 3, + "ramPerContainer": 256 + } + expected = { + 'falcon-startup.properties': { + 'properties': { + '*.application.services': 'foo,org.apache.falcon.atlas.service.AtlasService', + } + } + } + services = { + "services": [ + { + "href": "/api/v1/stacks/HDP/versions/2.2/services/ATLAS", + "StackServices": { + "service_name": "ATLAS", + "service_version": "2.6.0.2.2", + "stack_name": "HDP", + "stack_version": "2.3" + }, + "components": [ + { + "StackServiceComponents": { + "advertise_version": "false", + "cardinality": "1", + "component_category": "MASTER", + "component_name": "ATLAS_SERVER", + "display_name": "Atlas Server", + "is_client": "false", + "is_master": "true", + "hostnames": [] + }, + "dependencies": [] + } + ] + }, + ], + "configurations": { + "falcon-startup.properties": { + "properties": { + "*.application.services": "foo" + } + } + }, + "changed-configurations": [ ] + + } + hosts = { + "items" : [ + { + "href" : "/api/v1/hosts/c6401.ambari.apache.org", + "Hosts" : { + "cpu_count" : 1, + "host_name" : "c6401.ambari.apache.org", + "os_arch" : "x86_64", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "c6401.ambari.apache.org", + "rack_info" : "/default-rack", + "total_mem" : 1922680 + } + } + ] + } + + self.stackAdvisor.recommendFalconConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations, expected) + + services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.7.3_23'} + self.stackAdvisor.recommendFalconConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations, expected) + + def test_recommendSqoopConfigurations(self): + self.maxDiff = None + configurations = { + "sqoop-site": { + "properties": { + "sqoop.job.data.publish.class": "foo" + } + } + } + clusterData = { + "cpu": 4, + "mapMemory": 3000, + "amMemory": 2000, + "reduceMemory": 2056, + "containers": 3, + "ramPerContainer": 256 + } + expected = { + 'sqoop-site': { + 'properties': { + 'sqoop.job.data.publish.class': 'org.apache.atlas.sqoop.hook.SqoopHook', + } + } + } + services = { + "services": [ + { + "href": "/api/v1/stacks/HDP/versions/2.2/services/ATLAS", + "StackServices": { + "service_name": "ATLAS", + "service_version": "2.6.0.2.2", + "stack_name": "HDP", + "stack_version": "2.3" + }, + "components": [ + { + "StackServiceComponents": { + "advertise_version": "false", + "cardinality": "1", + "component_category": "MASTER", + "component_name": "ATLAS_SERVER", + "display_name": "Atlas Server", + "is_client": "false", + "is_master": "true", + "hostnames": [] + }, + "dependencies": [] + } + ] + }, + ], + "configurations": { + "sqoop-site": { + "properties": { + "sqoop.job.data.publish.class": "foo" + } + } + }, + "changed-configurations": [ ] + + } + hosts = { + "items" : [ + { + "href" : "/api/v1/hosts/c6401.ambari.apache.org", + "Hosts" : { + "cpu_count" : 1, + "host_name" : "c6401.ambari.apache.org", + "os_arch" : "x86_64", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "c6401.ambari.apache.org", + "rack_info" : "/default-rack", + "total_mem" : 1922680 + } + } + ] + } + + self.stackAdvisor.recommendSqoopConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations, expected) + + services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.7.3_23'} + self.stackAdvisor.recommendSqoopConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations, expected)
