Repository: ambari Updated Branches: refs/heads/branch-2.4 2e530e55e -> e505f0b24
AMBARI-17573. Add atlas-application config sections to all services that run Atlas hook, e.g., Hive, Falcon, Storm, Sqoop (alejandro) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e505f0b2 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e505f0b2 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e505f0b2 Branch: refs/heads/branch-2.4 Commit: e505f0b2419e7cee9ea6cad7a8c97de3943aa7e2 Parents: 2e530e5 Author: Alejandro Fernandez <[email protected]> Authored: Wed Jul 6 12:57:30 2016 -0700 Committer: Alejandro Fernandez <[email protected]> Committed: Wed Jul 6 13:12:28 2016 -0700 ---------------------------------------------------------------------- .../libraries/functions/setup_atlas_hook.py | 89 ++++++++++++++++++++ .../configuration/application-properties.xml | 42 ++------- .../falcon-atlas-application.properties.xml | 61 ++++++++++++++ .../FALCON/0.5.0.2.1/metainfo.xml | 3 + .../FALCON/0.5.0.2.1/package/scripts/falcon.py | 8 +- .../0.5.0.2.1/package/scripts/params_linux.py | 35 +++++--- .../package/scripts/setup_atlas_falcon.py | 41 --------- .../hive-atlas-application.properties.xml | 61 ++++++++++++++ .../HIVE/0.12.0.2.0/metainfo.xml | 6 ++ .../HIVE/0.12.0.2.0/package/scripts/hcat.py | 9 +- .../HIVE/0.12.0.2.0/package/scripts/hive.py | 7 +- .../0.12.0.2.0/package/scripts/params_linux.py | 17 ++-- .../package/scripts/setup_atlas_hive.py | 41 --------- .../HIVE/0.12.0.2.0/package/scripts/webhcat.py | 9 +- .../SQOOP/1.4.4.2.0/metainfo.xml | 5 ++ .../1.4.4.2.0/package/scripts/params_linux.py | 22 ++--- .../package/scripts/setup_atlas_sqoop.py | 52 ------------ .../SQOOP/1.4.4.2.0/package/scripts/sqoop.py | 14 ++- .../STORM/0.9.1/package/scripts/params_linux.py | 22 ++--- .../0.9.1/package/scripts/setup_atlas_storm.py | 53 ------------ .../STORM/0.9.1/package/scripts/storm.py | 10 ++- .../storm-atlas-application.properties.xml | 31 +++++++ .../common-services/STORM/1.0.1/metainfo.xml | 21 +++++ .../configuration/application-properties.xml | 42 +++++++++ 24 files changed, 418 insertions(+), 283 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/e505f0b2/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py new file mode 100644 index 0000000..872147a --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +__all__ = ["has_atlas_in_cluster", "setup_atlas_hook", "setup_atlas_jar_symlinks"] + +# Python Imports +import os + +# Local Imports +from resource_management.libraries.resources.properties_file import PropertiesFile +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.default import default +from resource_management.core.resources.system import Link +from resource_management.core.logger import Logger + + +def has_atlas_in_cluster(): + """ + Determine if Atlas is installed on the cluster. + :return: True if Atlas is installed, otherwise false. + """ + atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', []) + return len(atlas_hosts) > 0 + +def setup_atlas_hook(service_props, atlas_hook_filepath, owner, group): + """ + Generate the atlas-application.properties.xml file by merging the service_props with the Atlas application-properties. + :param service_props: Atlas configs specific to this service that must be merged. + :param atlas_hook_filepath: Config file to write, e.g., /etc/falcon/conf/atlas-application.properties.xml + :param owner: File owner + :param group: File group + """ + import params + atlas_props = default('/configurations/application-properties', {}) + + if has_atlas_in_cluster(): + merged_props = atlas_props.copy() + merged_props.update(service_props) + + Logger.info(format("Generating Atlas Hook config file {atlas_hook_filepath}")) + PropertiesFile(atlas_hook_filepath, + properties = merged_props, + owner = owner, + group = group, + mode = 0644) + + +def setup_atlas_jar_symlinks(hook_name, jar_source_dir): + """ + If atlas is present on this host, then link the jars from + {stack_root}/current/{hook_name}/lib/name_version.jar -> {jar_source_dir}/name_version.jar + @param hook_name: one of sqoop, storm + @param jar_source_dir: directory of where the symlinks need to be created from. + """ + import params + + if has_atlas_in_cluster(): + atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ \ + else format("{stack_root}/current/atlas-server") + + # Will only exist if this host contains Atlas Server + atlas_hook_dir = os.path.join(atlas_home_dir, "hook", hook_name) + if os.path.exists(atlas_hook_dir): + Logger.info("Atlas Server is present on this host, will symlink jars inside of %s to %s if not already done." % + (jar_source_dir, atlas_hook_dir)) + + src_files = os.listdir(atlas_hook_dir) + for file_name in src_files: + atlas_hook_file_name = os.path.join(atlas_hook_dir, file_name) + source_lib_file_name = os.path.join(jar_source_dir, file_name) + if os.path.isfile(atlas_hook_file_name): + Link(source_lib_file_name, to=atlas_hook_file_name) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e505f0b2/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml index 3578d43..b1b945d 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml @@ -158,12 +158,15 @@ <description>Indicates whether or not the notification service should be embedded.</description> <on-ambari-upgrade add="true"/> </property> + + <!-- atlas.cluster.name is also part of Atlas Hooks --> <property> <name>atlas.cluster.name</name> <value>{{cluster_name}}</value> <description>The cluster name.</description> <on-ambari-upgrade add="true"/> </property> + <property> <name>atlas.server.http.port</name> <value>21000</value> @@ -182,42 +185,9 @@ <description/> <on-ambari-upgrade add="true"/> </property> - <property> - <name>atlas.hook.falcon.synchronous</name> - <value>false</value> - <description/> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>atlas.hook.falcon.numRetries</name> - <value>3</value> - <description/> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>atlas.hook.falcon.minThreads</name> - <value>5</value> - <description/> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>atlas.hook.falcon.maxThreads</name> - <value>5</value> - <description/> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>atlas.hook.falcon.keepAliveTime</name> - <value>10</value> - <description/> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>atlas.hook.falcon.queueSize</name> - <value>1000</value> - <description/> - <on-ambari-upgrade add="true"/> - </property> + + + <property> <name>atlas.audit.hbase.tablename</name> <value>ATLAS_ENTITY_AUDIT_EVENTS</value> http://git-wip-us.apache.org/repos/asf/ambari/blob/e505f0b2/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-atlas-application.properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-atlas-application.properties.xml b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-atlas-application.properties.xml new file mode 100644 index 0000000..4811910 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-atlas-application.properties.xml @@ -0,0 +1,61 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false"> + <!-- These are the Atlas Hooks properties specific to this service. This file is then merged with common properties + that apply to all services. --> + <property> + <name>atlas.hook.falcon.synchronous</name> + <value>false</value> + <description/> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>atlas.hook.falcon.numRetries</name> + <value>3</value> + <description/> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>atlas.hook.falcon.minThreads</name> + <value>5</value> + <description/> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>atlas.hook.falcon.maxThreads</name> + <value>5</value> + <description/> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>atlas.hook.falcon.keepAliveTime</name> + <value>10</value> + <description/> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>atlas.hook.falcon.queueSize</name> + <value>1000</value> + <description/> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/e505f0b2/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/metainfo.xml b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/metainfo.xml index 602144b..40334d6 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/metainfo.xml @@ -115,10 +115,13 @@ <config-type>falcon-startup.properties</config-type> <config-type>falcon-runtime.properties</config-type> <config-type>falcon-client.properties</config-type> + <config-type>falcon-atlas-application.properties</config-type> </configuration-dependencies> + <!-- Make a new config version when these config types are changed. --> <excluded-config-types> <config-type>oozie-site</config-type> + <config-type>application.properties</config-type> </excluded-config-types> <quickLinksConfigurations> http://git-wip-us.apache.org/repos/asf/ambari/blob/e505f0b2/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py index a67519d..6111c34 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py @@ -19,6 +19,7 @@ limitations under the License. import os.path +# Local Imports from resource_management.core.environment import Environment from resource_management.core.source import InlineTemplate from resource_management.core.source import Template @@ -31,12 +32,12 @@ from resource_management.libraries.script import Script from resource_management.libraries.resources import PropertiesFile from resource_management.libraries.functions import format from resource_management.libraries.functions.show_logs import show_logs +from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster, setup_atlas_hook from resource_management.libraries.functions import get_user_call_output from resource_management.core.logger import Logger from ambari_commons import OSConst from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl -from setup_atlas_falcon import setup_atlas_falcon @OsFamilyFuncImpl(os_family = OsFamilyImpl.DEFAULT) def falcon(type, action = None, upgrade_type=None): @@ -110,7 +111,10 @@ def falcon(type, action = None, upgrade_type=None): create_parents = True, cd_access = "a") - setup_atlas_falcon() + # Generate atlas-application.properties.xml file + if has_atlas_in_cluster(): + atlas_hook_filepath = os.path.join(params.falcon_conf_dir, params.atlas_hook_filename) + setup_atlas_hook(params.falcon_atlas_application_properties, atlas_hook_filepath, params.falcon_user, params.user_group) if type == 'server': if action == 'config': http://git-wip-us.apache.org/repos/asf/ambari/blob/e505f0b2/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py index fc9d8b9..e58d205 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py @@ -30,6 +30,7 @@ from resource_management.libraries.functions.expect import expect from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster config = Script.get_config() stack_root = status_params.stack_root @@ -133,17 +134,31 @@ supports_falcon_extensions = (stack_version_formatted and check_stack_feature(St local_data_mirroring_dir = format('{stack_root}/current/falcon-server/data-mirroring') dfs_data_mirroring_dir = "/apps/data-mirroring" +######################################################## +############# Atlas related params ##################### +######################################################## +#region Atlas Hooks +falcon_atlas_application_properties = default('/configurations/falcon-atlas-application.properties', {}) + +# Calculate atlas_hook_cp to add to FALCON_EXTRA_CLASS_PATH atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', []) -has_atlas = len(atlas_hosts) > 0 -atlas_plugin_package = "atlas-metadata*-hive-plugin" -atlas_ubuntu_plugin_package = "atlas-metadata.*-hive-plugin" - -if has_atlas: - atlas_conf_file = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties') - atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf' - atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else format('{stack_root}/current/atlas-server') - atlas_hook_cp = atlas_conf_dir + os.pathsep + os.path.join(atlas_home_dir, "hook", "falcon", "*") + os.pathsep - atlas_props = default('/configurations/application-properties', {}) +has_atlas_server_on_host = False + +for host in atlas_hosts: + if host.upper() == hostname.upper(): + has_atlas_server_on_host = True + break + +# Path to add to environment variable +atlas_hook_cp = "" +if has_atlas_in_cluster(): + atlas_hook_filename = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties') + + if has_atlas_server_on_host: + atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else format('{stack_root}/current/atlas-server/conf') + atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else format('{stack_root}/current/atlas-server') + atlas_hook_cp = atlas_conf_dir + os.pathsep + os.path.join(atlas_home_dir, "hook", "falcon", "*") + os.pathsep +#endregion hdfs_site = config['configurations']['hdfs-site'] default_fs = config['configurations']['core-site']['fs.defaultFS'] http://git-wip-us.apache.org/repos/asf/ambari/blob/e505f0b2/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/setup_atlas_falcon.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/setup_atlas_falcon.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/setup_atlas_falcon.py deleted file mode 100644 index 1dce515..0000000 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/setup_atlas_falcon.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management.libraries.resources.properties_file import PropertiesFile -from resource_management.core.resources.packaging import Package -from resource_management.libraries.functions.format import format -from ambari_commons import OSCheck - -import os - -def setup_atlas_falcon(): - import params - - if params.has_atlas: - - if not params.host_sys_prepped: - Package(params.atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else params.atlas_plugin_package, - retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count) - - PropertiesFile(format('{falcon_conf_dir}/{atlas_conf_file}'), - properties = params.atlas_props, - owner = params.falcon_user, - group = params.user_group, - mode = 0644) http://git-wip-us.apache.org/repos/asf/ambari/blob/e505f0b2/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-atlas-application.properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-atlas-application.properties.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-atlas-application.properties.xml new file mode 100644 index 0000000..9022d0a --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-atlas-application.properties.xml @@ -0,0 +1,61 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false"> + <!-- These are the Atlas Hooks properties specific to this service. This file is then merged with common properties + that apply to all services. --> + <property> + <name>atlas.hook.hive.synchronous</name> + <value>false</value> + <description/> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>atlas.hook.hive.numRetries</name> + <value>3</value> + <description/> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>atlas.hook.hive.minThreads</name> + <value>5</value> + <description/> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>atlas.hook.hive.maxThreads</name> + <value>5</value> + <description/> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>atlas.hook.hive.keepAliveTime</name> + <value>10</value> + <description/> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>atlas.hook.hive.queueSize</name> + <value>1000</value> + <description/> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/e505f0b2/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/metainfo.xml index 273133a..0dcef40 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/metainfo.xml @@ -344,7 +344,13 @@ <config-type>ranger-hive-policymgr-ssl</config-type> <config-type>ranger-hive-security</config-type> <config-type>mapred-site</config-type> + <config-type>hive-atlas-application.properties</config-type> </configuration-dependencies> + + <!-- Make a new config version when these config types are changed. --> + <excluded-config-types> + <config-type>application.properties</config-type> + </excluded-config-types> </service> </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/e505f0b2/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat.py index 45dcbe7..f53625c 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat.py @@ -17,11 +17,13 @@ See the License for the specific language governing permissions and limitations under the License. """ +import os from resource_management import * import sys from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from ambari_commons import OSConst +from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster, setup_atlas_hook @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) @@ -40,8 +42,6 @@ def hcat(): def hcat(): import params - from setup_atlas_hive import setup_atlas_hive - Directory(params.hive_conf_dir, create_parents = True, owner=params.hcat_user, @@ -74,4 +74,7 @@ def hcat(): content=InlineTemplate(params.hcat_env_sh_template) ) - setup_atlas_hive() + # Generate atlas-application.properties.xml file + if has_atlas_in_cluster(): + atlas_hook_filepath = os.path.join(params.hive_config_dir, params.atlas_hook_filename) + setup_atlas_hook(params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group) http://git-wip-us.apache.org/repos/asf/ambari/blob/e505f0b2/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py index ea2af62..6b05134 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py @@ -39,11 +39,11 @@ from resource_management.core.shell import as_sudo from resource_management.core.shell import quote_bash_args from resource_management.core.logger import Logger from resource_management.core import utils +from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster, setup_atlas_hook from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from ambari_commons import OSConst -from setup_atlas_hive import setup_atlas_hive @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) @@ -227,7 +227,10 @@ def hive(name=None): group=params.user_group, mode=0644) - setup_atlas_hive() + # Generate atlas-application.properties.xml file + if has_atlas_in_cluster(): + atlas_hook_filepath = os.path.join(params.hive_config_dir, params.atlas_hook_filename) + setup_atlas_hook(params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group) if name == 'hiveserver2': XmlConfig("hiveserver2-site.xml", http://git-wip-us.apache.org/repos/asf/ambari/blob/e505f0b2/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index 200344d..571eebd 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -40,6 +40,7 @@ from resource_management.libraries.functions.stack_features import check_stack_f from resource_management.libraries.functions.get_port_from_url import get_port_from_url from resource_management.libraries.functions.expect import expect from resource_management.libraries import functions +from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster # Default log4j version; put config files under /etc/hive/conf log4j_version = '1' @@ -505,18 +506,12 @@ metrics_collection_period = default("/configurations/ams-site/timeline.metrics.s ######################################################## ############# Atlas related params ##################### ######################################################## +#region Atlas Hooks +hive_atlas_application_properties = default('/configurations/hive-atlas-application.properties', {}) -atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', []) -has_atlas = len(atlas_hosts) > 0 -classpath_addition = "" -atlas_plugin_package = "atlas-metadata*-hive-plugin" -atlas_ubuntu_plugin_package = "atlas-metadata.*-hive-plugin" - -if has_atlas: - atlas_conf_file = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties') - atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else format('{stack_root}/current/atlas-server') - atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf' - atlas_props = default('/configurations/application-properties', {}) +if has_atlas_in_cluster(): + atlas_hook_filename = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties') +#endregion ######################################################## ########### WebHCat related params ##################### http://git-wip-us.apache.org/repos/asf/ambari/blob/e505f0b2/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_atlas_hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_atlas_hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_atlas_hive.py deleted file mode 100644 index d1bd8ea..0000000 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_atlas_hive.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management.libraries.resources.properties_file import PropertiesFile -from resource_management.core.resources.packaging import Package -from resource_management.libraries.functions.format import format -from ambari_commons import OSCheck - -def setup_atlas_hive(configuration_directory=None): - import params - - if params.has_atlas: - if configuration_directory is None: - configuration_directory = format("{hive_config_dir}") - - if not params.host_sys_prepped: - Package(params.atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else params.atlas_plugin_package, # FIXME HACK: install the package during RESTART/START when install_packages is not triggered. - retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count) - - PropertiesFile(format('{configuration_directory}/{atlas_conf_file}'), - properties = params.atlas_props, - owner = params.hive_user, - group = params.user_group, - mode = 0644) http://git-wip-us.apache.org/repos/asf/ambari/blob/e505f0b2/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat.py index 3acbc7b..4f38055 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat.py @@ -25,6 +25,7 @@ from resource_management.core.resources.system import Execute from resource_management.libraries.functions import StackFeature from resource_management.libraries.functions.stack_features import check_stack_feature from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster, setup_atlas_hook from ambari_commons import OSConst @@ -46,8 +47,6 @@ def webhcat(): def webhcat(): import params - from setup_atlas_hive import setup_atlas_hive - Directory(params.templeton_pid_dir, owner=params.webhcat_user, mode=0755, @@ -138,4 +137,8 @@ def webhcat(): content=StaticFile(format("{config_dir}/{log4j_webhcat_filename}.template")) ) - setup_atlas_hive(configuration_directory=params.config_dir) + # Generate atlas-application.properties.xml file + if has_atlas_in_cluster(): + # WebHCat uses a different config dir than the rest of the daemons in Hive. + atlas_hook_filepath = os.path.join(params.config_dir, params.atlas_hook_filename) + setup_atlas_hook(params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group) http://git-wip-us.apache.org/repos/asf/ambari/blob/e505f0b2/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml index e3aa5ef..7b2b1a2 100644 --- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml @@ -96,6 +96,11 @@ <config-type>sqoop-env</config-type> <config-type>sqoop-site</config-type> </configuration-dependencies> + + <!-- Make a new config version when these config types are changed. --> + <excluded-config-types> + <config-type>application.properties</config-type> + </excluded-config-types> </service> </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/e505f0b2/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py index b2a6802..5bca720 100644 --- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py @@ -17,6 +17,8 @@ limitations under the License. """ +import os + from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions.default import default from resource_management.libraries.functions.get_kinit_path import get_kinit_path @@ -25,7 +27,8 @@ from resource_management.libraries.functions.format import format from resource_management.libraries.functions import StackFeature from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions.expect import expect -import os +from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster + # a map of the Ambari role to the component name # for use with <stack-root>/current/<component> @@ -120,20 +123,13 @@ if "jdbc_drivers" in config['configurations']['sqoop-env']: sqoop_jdbc_drivers_name_dict[jdbc_name] = jdbc_driver_name jdk_location = config['hostLevelParams']['jdk_location'] -job_data_publish_class = '' ######################################################## ############# Atlas related params ##################### ######################################################## +#region Atlas Hooks +sqoop_atlas_application_properties = default('/configurations/sqoop-atlas-application.properties', {}) -atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', []) -has_atlas = len(atlas_hosts) > 0 -atlas_plugin_package = "atlas-metadata*-hive-plugin" -atlas_ubuntu_plugin_package = "atlas-metadata.*-hive-plugin" - -if has_atlas: - atlas_conf_file = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties') - atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else format("{stack_root}/current/atlas-server") - atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf' - atlas_props = default('/configurations/application-properties', {}) - job_data_publish_class = 'org.apache.atlas.sqoop.hook.SqoopHook' +if has_atlas_in_cluster(): + atlas_hook_filename = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties') +#endregion http://git-wip-us.apache.org/repos/asf/ambari/blob/e505f0b2/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/setup_atlas_sqoop.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/setup_atlas_sqoop.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/setup_atlas_sqoop.py deleted file mode 100644 index 76c1cda..0000000 --- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/setup_atlas_sqoop.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management.libraries.resources.properties_file import PropertiesFile -from resource_management.core.resources.packaging import Package -from resource_management.core.resources.system import Link -from resource_management.libraries.functions.format import format -from ambari_commons import OSCheck - -import os - -def setup_atlas_sqoop(): - import params - - if params.has_atlas: - - if not params.host_sys_prepped: - Package(params.atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else params.atlas_plugin_package, - retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count) - - PropertiesFile(format('{sqoop_conf_dir}/{atlas_conf_file}'), - properties = params.atlas_props, - owner = params.sqoop_user, - group = params.user_group, - mode = 0644) - - atlas_sqoop_hook_dir = os.path.join(params.atlas_home_dir, "hook", "sqoop") - if os.path.exists(atlas_sqoop_hook_dir): - - src_files = os.listdir(atlas_sqoop_hook_dir) - for file_name in src_files: - atlas_sqoop_hook_file_name = os.path.join(atlas_sqoop_hook_dir, file_name) - sqoop_lib_file_name = os.path.join(params.sqoop_lib, file_name) - if (os.path.isfile(atlas_sqoop_hook_file_name)): - Link(sqoop_lib_file_name, to = atlas_sqoop_hook_file_name) http://git-wip-us.apache.org/repos/asf/ambari/blob/e505f0b2/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py index bac836c..963d169 100644 --- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py @@ -16,16 +16,17 @@ See the License for the specific language governing permissions and limitations under the License. """ +# Python Imports +import os +# Local Imports from resource_management.core.source import InlineTemplate, DownloadSource from resource_management.libraries.functions import format from resource_management.libraries.resources.xml_config import XmlConfig from resource_management.core.resources.system import File, Link, Directory from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from ambari_commons import OSConst -from setup_atlas_sqoop import setup_atlas_sqoop - -import os +from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster, setup_atlas_hook, setup_atlas_jar_symlinks @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) @@ -61,7 +62,12 @@ def sqoop(type=None): group = params.user_group ) - setup_atlas_sqoop() + # Generate atlas-application.properties.xml file and symlink the hook jars + if has_atlas_in_cluster(): + atlas_hook_filepath = os.path.join(params.sqoop_conf_dir, params.atlas_hook_filename) + setup_atlas_hook(params.sqoop_atlas_application_properties, atlas_hook_filepath, params.sqoop_user, params.user_group) + setup_atlas_jar_symlinks("sqoop", params.sqoop_lib) + File(format("{sqoop_conf_dir}/sqoop-env.sh"), owner=params.sqoop_user, http://git-wip-us.apache.org/repos/asf/ambari/blob/e505f0b2/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py index 31e49e0..93d4d4e 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py @@ -38,6 +38,7 @@ from resource_management.libraries.functions.get_not_managed_resources import ge from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions import StackFeature from resource_management.libraries.functions.expect import expect +from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster # server configurations config = Script.get_config() @@ -193,18 +194,19 @@ metric_collector_legacy_sink_jar = "/usr/lib/storm/lib/ambari-metrics-storm-sink jar_jvm_opts = '' -# Atlas related params -atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', []) -has_atlas = len(atlas_hosts) > 0 -atlas_plugin_package = "atlas-metadata*-hive-plugin" -atlas_ubuntu_plugin_package = "atlas-metadata.*-hive-plugin" -if has_atlas: - atlas_conf_file = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties') - atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else stack_root + '/current/atlas-server' +######################################################## +############# Atlas related params ##################### +######################################################## +#region Atlas Hooks +storm_atlas_application_properties = default('/configurations/storm-atlas-application.properties', {}) + +if has_atlas_in_cluster(): + atlas_hook_filename = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties') atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf' - atlas_props = default('/configurations/application-properties', {}) - jar_jvm_opts = '-Datlas.conf=' + atlas_conf_dir + jar_jvm_opts += '-Datlas.conf=' + atlas_conf_dir +#endregion + # ranger host stack_supports_ranger_audit_db = stack_version_formatted and check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, stack_version_formatted) http://git-wip-us.apache.org/repos/asf/ambari/blob/e505f0b2/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/setup_atlas_storm.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/setup_atlas_storm.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/setup_atlas_storm.py deleted file mode 100644 index 76ca1d3..0000000 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/setup_atlas_storm.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management.libraries.resources.properties_file import PropertiesFile -from resource_management.core.resources.packaging import Package -from resource_management.core.resources.system import Link -from resource_management.libraries.functions.format import format -from ambari_commons import OSCheck - -import os - -def setup_atlas_storm(): - import params - - if params.has_atlas: - - if not params.host_sys_prepped: - Package(params.atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else params.atlas_plugin_package, - retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count) - - PropertiesFile(format('{conf_dir}/{atlas_conf_file}'), - properties = params.atlas_props, - owner = params.storm_user, - group = params.user_group, - mode = 0644) - - atlas_storm_hook_dir = os.path.join(params.atlas_home_dir, "hook", "storm") - if os.path.exists(atlas_storm_hook_dir): - storm_extlib_dir = os.path.join(params.storm_component_home_dir, "extlib") - if os.path.exists(storm_extlib_dir): - src_files = os.listdir(atlas_storm_hook_dir) - for file_name in src_files: - atlas_storm_hook_file_name = os.path.join(atlas_storm_hook_dir, file_name) - storm_lib_file_name = os.path.join(storm_extlib_dir, file_name) - if (os.path.isfile(atlas_storm_hook_file_name)): - Link(storm_lib_file_name, to = atlas_storm_hook_file_name) http://git-wip-us.apache.org/repos/asf/ambari/blob/e505f0b2/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/storm.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/storm.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/storm.py index db80e3a..bce71ca 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/storm.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/storm.py @@ -31,7 +31,8 @@ from resource_management.libraries.functions import StackFeature from storm_yaml_utils import yaml_config_template, yaml_config from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from ambari_commons import OSConst -from setup_atlas_storm import setup_atlas_storm +from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster, setup_atlas_hook, setup_atlas_jar_symlinks + @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) def storm(name=None): @@ -96,7 +97,12 @@ def storm(name=None): content=InlineTemplate(params.storm_env_sh_template) ) - setup_atlas_storm() + # Generate atlas-application.properties.xml file and symlink the hook jars + if has_atlas_in_cluster(): + atlas_hook_filepath = os.path.join(params.conf_dir, params.atlas_hook_filename) + setup_atlas_hook(params.storm_atlas_application_properties, atlas_hook_filepath, params.storm_user, params.user_group) + storm_extlib_dir = os.path.join(params.storm_component_home_dir, "extlib") + setup_atlas_jar_symlinks("storm", storm_extlib_dir) if params.has_metric_collector: File(format("{conf_dir}/storm-metrics2.properties"), http://git-wip-us.apache.org/repos/asf/ambari/blob/e505f0b2/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-atlas-application.properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-atlas-application.properties.xml b/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-atlas-application.properties.xml new file mode 100644 index 0000000..47d7758 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-atlas-application.properties.xml @@ -0,0 +1,31 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false"> + <!-- These are the Atlas Hooks properties specific to this service. This file is then merged with common properties + that apply to all services. --> + <property> + <name>atlas.hook.storm.numRetries</name> + <value>3</value> + <description/> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/e505f0b2/ambari-server/src/main/resources/common-services/STORM/1.0.1/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/1.0.1/metainfo.xml b/ambari-server/src/main/resources/common-services/STORM/1.0.1/metainfo.xml index 3c54a1e..a5695d8 100644 --- a/ambari-server/src/main/resources/common-services/STORM/1.0.1/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/STORM/1.0.1/metainfo.xml @@ -23,6 +23,27 @@ <name>STORM</name> <version>1.0.1</version> <extends>common-services/STORM/0.10.0</extends> + + <configuration-dependencies> + <config-type>storm-site</config-type> + <config-type>storm-env</config-type> + <config-type>ranger-storm-plugin-properties</config-type> + <config-type>ranger-storm-audit</config-type> + <config-type>ranger-storm-policymgr-ssl</config-type> + <config-type>ranger-storm-security</config-type> + <config-type>admin-properties</config-type> + <config-type>ranger-ugsync-site</config-type> + <config-type>ranger-admin-site</config-type> + <config-type>zookeeper-env</config-type> + <config-type>zoo.cfg</config-type> + <config-type>ams-ssl-client</config-type> + <config-type>storm-atlas-application.properties</config-type> + </configuration-dependencies> + + <!-- Make a new config version when these config types are changed. --> + <excluded-config-types> + <config-type>application.properties</config-type> + </excluded-config-types> </service> </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/e505f0b2/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml index 4c7e2c6..57b31a7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml @@ -145,6 +145,10 @@ <description>The Kafka data directory.</description> <on-ambari-upgrade add="true"/> </property> + + <!-- Start: Atlas Hooks that are also written out to configs for Falcon, Storm, Hive, and Sqoop. + There are several more properties for when Atlas is Kerberized. + --> <property> <name>atlas.kafka.bootstrap.servers</name> <value/> @@ -164,6 +168,44 @@ <on-ambari-upgrade add="true"/> </property> <property> + <name>atlas.kafka.zookeeper.session.timeout.ms</name> + <value>400</value> + <description></description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>atlas.kafka.zookeeper.connection.timeout.ms</name> + <value>200</value> + <description></description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>atlas.kafka.zookeeper.sync.time.ms</name> + <value>20</value> + <description></description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>atlas.notification.create.topics</name> + <value>true</value> + <description></description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>atlas.notification.replicas</name> + <value>1</value> + <description></description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>atlas.notification.topics</name> + <value>ATLAS_HOOK,ATLAS_ENTITIES</value> + <description></description> + <on-ambari-upgrade add="true"/> + </property> + <!-- End: Atlas Hooks --> + + <property> <name>atlas.kafka.entities.group.id</name> <value>entities</value> <description>Kafka group id for the entity topic.</description>
