Repository: ambari Updated Branches: refs/heads/trunk e195171f3 -> bf04d16b1
AMBARI-14383. Add support for Ranger TagSync process as a component under RANGER(gautam) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/bf04d16b Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/bf04d16b Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/bf04d16b Branch: refs/heads/trunk Commit: bf04d16b103e89a2d6a0d4538b47a99d3d75a772 Parents: e195171 Author: Gautam Borad <[email protected]> Authored: Mon Apr 4 11:19:05 2016 +0530 Committer: Gautam Borad <[email protected]> Committed: Tue Apr 5 09:21:04 2016 +0530 ---------------------------------------------------------------------- .../libraries/functions/conf_select.py | 6 + .../libraries/functions/constants.py | 1 + .../libraries/functions/package_conditions.py | 11 +- .../libraries/functions/stack_features.py | 5 + .../libraries/functions/stack_select.py | 2 + .../RANGER/0.4.0/package/scripts/params.py | 24 +- .../0.4.0/package/scripts/ranger_service.py | 12 +- .../0.4.0/package/scripts/ranger_tagsync.py | 81 +++++ .../0.4.0/package/scripts/setup_ranger_xml.py | 101 +++++- .../0.4.0/package/scripts/status_params.py | 4 +- .../HDP/2.0.6/properties/stack_features.json | 5 + .../stacks/HDP/2.2/services/RANGER/metainfo.xml | 8 + .../configuration/ranger-tagsync-site.xml | 187 +++++++++++ .../tagsync-application-properties.xml | 95 ++++++ .../stacks/HDP/2.5/services/RANGER/metainfo.xml | 30 +- .../services/RANGER/themes/theme_version_3.json | 308 +++++++++++++++++++ .../stacks/HDP/2.5/services/stack_advisor.py | 36 ++- .../stacks/HDP/2.6/services/RANGER/metainfo.xml | 2 +- 18 files changed, 907 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/bf04d16b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py index a85befb..b5de69d 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py @@ -115,6 +115,12 @@ _PACKAGE_DIRS = { "current_dir": "{0}/current/ranger-admin/conf".format(STACK_ROOT_PATTERN) } ], + "ranger-tagsync": [ + { + "conf_dir": "/etc/ranger/tagsync/conf", + "current_dir": "{0}/current/ranger-tagsync/conf".format(STACK_ROOT_PATTERN) + } + ], "ranger-kms": [ { "conf_dir": "/etc/ranger/kms/conf", http://git-wip-us.apache.org/repos/asf/ambari/blob/bf04d16b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py index f7d817a..055a576 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py @@ -47,6 +47,7 @@ class StackFeature: DATANODE_NON_ROOT = "datanode_non_root" REMOVE_RANGER_HDFS_PLUGIN_ENV = "remove_ranger_hdfs_plugin_env" RANGER = "ranger" + RANGER_TAGSYNC_COMPONENT = "ranger_tagsync_component" PHOENIX = "phoenix" NFS = "nfs" TEZ_FOR_SPARK = "tez_for_spark" http://git-wip-us.apache.org/repos/asf/ambari/blob/bf04d16b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py index 4ca3b7b..df46ce0 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py @@ -20,7 +20,7 @@ Ambari Agent """ __all__ = ["is_lzo_enabled", "should_install_phoenix", "should_install_ams_collector", "should_install_ams_grafana", - "should_install_mysql", "should_install_mysl_connector"] + "should_install_mysql", "should_install_mysl_connector", "should_install_ranger_tagsync"] import os from resource_management.libraries.script import Script @@ -66,4 +66,11 @@ def should_install_hive_atlas(): def should_install_kerberos_server(): config = Script.get_config() - return 'role' in config and config['role'] != "KERBEROS_CLIENT" \ No newline at end of file + return 'role' in config and config['role'] != "KERBEROS_CLIENT" + +def should_install_ranger_tagsync(): + config = Script.get_config() + ranger_tagsync_hosts = default("/clusterHostInfo/ranger_tagsync_hosts", []) + has_ranger_tagsync = len(ranger_tagsync_hosts) > 0 + + return has_ranger_tagsync \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/bf04d16b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py index 4fc3489..8d31fdc 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py @@ -65,6 +65,11 @@ _DEFAULT_STACK_FEATURES = { "min_version": "2.2.0.0" }, { + "name": "ranger_tagsync_component", + "description": "Ranger Tagsync component support (AMBARI-14383)", + "min_version": "2.5.0.0" + }, + { "name": "phoenix", "description": "Phoenix Service support", "min_version": "2.3.0.0" http://git-wip-us.apache.org/repos/asf/ambari/blob/bf04d16b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py index 6308a81..b994fce 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py @@ -64,6 +64,8 @@ SERVER_ROLE_DIRECTORY_MAP = { 'OOZIE_SERVER' : 'oozie-server', 'RANGER_ADMIN' : 'ranger-admin', 'RANGER_USERSYNC' : 'ranger-usersync', + 'RANGER_TAGSYNC' : 'ranger-tagsync', + 'RANGER_KMS' : 'ranger-kms', 'SPARK_JOBHISTORYSERVER' : 'spark-historyserver', 'SPARK_THRIFTSERVER' : 'spark-thriftserver', 'NIMBUS' : 'storm-nimbus', http://git-wip-us.apache.org/repos/asf/ambari/blob/bf04d16b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py index 31398f3..c0ab3e8 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py @@ -31,7 +31,8 @@ from resource_management.libraries.functions import StackFeature # for use with <stack-root>/current/<component> SERVER_ROLE_DIRECTORY_MAP = { 'RANGER_ADMIN' : 'ranger-admin', - 'RANGER_USERSYNC' : 'ranger-usersync' + 'RANGER_USERSYNC' : 'ranger-usersync', + 'RANGER_TAGSYNC' : 'ranger-tagsync' } component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "RANGER_ADMIN") @@ -56,12 +57,15 @@ create_db_dbuser = config['configurations']['ranger-env']['create_db_dbuser'] stack_supports_rolling_upgrade = stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted) stack_supports_config_versioning = stack_version_formatted and check_stack_feature(StackFeature.CONFIG_VERSIONING, stack_version_formatted) stack_supports_usersync_non_root = stack_version_formatted and check_stack_feature(StackFeature.RANGER_USERSYNC_NON_ROOT, stack_version_formatted) +stack_supports_ranger_tagsync = stack_version_formatted and check_stack_feature(StackFeature.RANGER_TAGSYNC_COMPONENT, stack_version_formatted) downgrade_from_version = default("/commandParams/downgrade_from_version", None) upgrade_direction = default("/commandParams/upgrade_direction", None) ranger_conf = '/etc/ranger/admin/conf' ranger_ugsync_conf = '/etc/ranger/usersync/conf' +ranger_tagsync_home = format('{stack_root}/current/ranger-tagsync') +ranger_tagsync_conf = format('{stack_root}/current/ranger-tagsync/conf') if upgrade_direction == Direction.DOWNGRADE and version and not check_stack_feature(StackFeature.CONFIG_VERSIONING, version): stack_supports_rolling_upgrade = True @@ -84,12 +88,17 @@ if stack_supports_config_versioning: ranger_conf = format('{stack_root}/current/ranger-admin/conf') ranger_ugsync_conf = format('{stack_root}/current/ranger-usersync/conf') +if stack_supports_ranger_tagsync: + ranger_tagsync_home = format('{stack_root}/current/ranger-tagsync') + tagsync_bin = '/usr/bin/ranger-tagsync' + ranger_tagsync_conf = format('{stack_root}/current/ranger-tagsync/conf') + usersync_services_file = format('{stack_root}/current/ranger-usersync/ranger-usersync-services.sh') java_home = config['hostLevelParams']['java_home'] unix_user = config['configurations']['ranger-env']['ranger_user'] unix_group = config['configurations']['ranger-env']['ranger_group'] -ranger_pid_dir = config['configurations']['ranger-env']['ranger_pid_dir'] +ranger_pid_dir = default("/configurations/ranger-env/ranger_pid_dir", "/var/run/ranger") usersync_log_dir = default("/configurations/ranger-env/ranger_usersync_log_dir", "/var/log/ranger/usersync") admin_log_dir = default("/configurations/ranger-env/ranger_admin_log_dir", "/var/log/ranger/admin") ranger_admin_default_file = format('{ranger_conf}/ranger-admin-default-site.xml') @@ -200,3 +209,14 @@ ug_sync_source = config["configurations"]["ranger-ugsync-site"]["ranger.usersync current_host = config['hostname'] if current_host in ranger_admin_hosts: ranger_host = current_host + +# ranger-tagsync +ranger_tagsync_hosts = default("/clusterHostInfo/ranger_tagsync_hosts", []) +has_ranger_tagsync = len(ranger_tagsync_hosts) > 0 + +tagsync_enabled = config["configurations"]["ranger-tagsync-site"]['ranger.tagsync.enabled'] +tagsync_log_dir = default("/configurations/ranger-tagsync-site/ranger.tagsync.logdir", "/var/log/ranger/tagsync") +ranger_tagsync_tagadmin_password = unicode(config["configurations"]["ranger-tagsync-site"]["ranger.tagsync.tagadmin.password"]) if has_ranger_tagsync else None +tagsync_jceks_path = config["configurations"]["ranger-tagsync-site"]["ranger.tagsync.tagadmin.keystore"] +tagsync_application_properties = dict(config["configurations"]["tagsync-application-properties"]) if has_ranger_tagsync else None +tagsync_pid_file = format('{ranger_pid_dir}/tagsync.pid') http://git-wip-us.apache.org/repos/asf/ambari/blob/bf04d16b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_service.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_service.py index 9477b8e..9f4757b 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_service.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_service.py @@ -42,7 +42,7 @@ def ranger_service(name, action=None): Execute(params.usersync_start, environment=env_dict, not_if=no_op_test, - user=params.unix_user, + user=params.unix_user ) except: show_logs(params.usersync_log_dir, params.unix_user) @@ -52,5 +52,13 @@ def ranger_service(name, action=None): Execute((params.usersync_start,), environment={'JAVA_HOME': params.java_home}, not_if=no_op_test, - sudo=True, + sudo=True ) + elif name == 'ranger_tagsync' and params.stack_supports_ranger_tagsync: + no_op_test = format('ps -ef | grep proc_rangertagsync | grep -v grep') + cmd = format('{tagsync_bin} start') + Execute(cmd, + environment=env_dict, + user=params.unix_user, + not_if=no_op_test + ) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/bf04d16b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_tagsync.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_tagsync.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_tagsync.py new file mode 100644 index 0000000..79316f9 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_tagsync.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from resource_management.libraries.script import Script +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.core.resources.system import Execute, File +from resource_management.libraries.functions.check_process_status import check_process_status +from resource_management.core.exceptions import ComponentIsNotRunning +from resource_management.libraries.functions.format import format +from resource_management.core.logger import Logger +from resource_management.core import shell +from ranger_service import ranger_service +from setup_ranger_xml import ranger +import upgrade + +class RangerTagsync(Script): + + def install(self, env): + self.install_packages(env) + self.configure(env) + + def configure(self, env, upgrade_type=None): + import params + env.set_params(params) + ranger('ranger_tagsync', upgrade_type=upgrade_type) + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + + self.configure(env, upgrade_type=upgrade_type) + ranger_service('ranger_tagsync') + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + + Execute(format('{tagsync_bin} stop'), environment={'JAVA_HOME': params.java_home}, user=params.unix_user) + File(params.tagsync_pid_file, + action = "delete" + ) + + def status(self, env): + import status_params + env.set_params(status_params) + + check_process_status(status_params.tagsync_pid_file) + + def pre_upgrade_restart(self, env, upgrade_type=None): + import params + env.set_params(params) + + if params.stack_supports_ranger_tagsync: + Logger.info("Executing Ranger Tagsync Stack Upgrade pre-restart") + conf_select.select(params.stack_name, "ranger-tagsync", params.version) + stack_select.select("ranger-tagsync", params.version) + + def get_stack_to_component(self): + import params + return {params.stack_name: "ranger-tagsync"} + + +if __name__ == "__main__": + RangerTagsync().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/bf04d16b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py index 9a856a5..9544ee8 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py @@ -24,6 +24,7 @@ from resource_management.core.resources.system import File, Directory, Execute, from resource_management.core.source import DownloadSource, InlineTemplate from resource_management.libraries.resources.xml_config import XmlConfig from resource_management.libraries.resources.modify_properties_file import ModifyPropertiesFile +from resource_management.libraries.resources.properties_file import PropertiesFile from resource_management.core.exceptions import Fail from resource_management.libraries.functions.format import format from resource_management.libraries.functions.is_empty import is_empty @@ -43,6 +44,9 @@ def ranger(name=None, upgrade_type=None): if name == 'ranger_usersync': setup_usersync(upgrade_type=upgrade_type) + if name == 'ranger_tagsync': + setup_tagsync(upgrade_type=upgrade_type) + def setup_ranger_admin(upgrade_type=None): import params @@ -349,7 +353,7 @@ def setup_usersync(upgrade_type=None): ) Directory(format("{ranger_ugsync_conf}/"), - owner = params.unix_user + owner = params.unix_user ) if upgrade_type is not None: @@ -424,3 +428,98 @@ def setup_usersync(upgrade_type=None): group = params.unix_group, mode = 0640 ) + +def setup_tagsync(upgrade_type=None): + import params + + ranger_tagsync_home = params.ranger_tagsync_home + ranger_home = params.ranger_home + ranger_tagsync_conf = params.ranger_tagsync_conf + + tagsync_log4j_file = format('{ranger_tagsync_conf}/log4j.xml') + tagsync_services_file = format('{ranger_tagsync_home}/ranger-tagsync-services.sh') + + Directory(format("{ranger_tagsync_conf}"), + owner = params.unix_user, + group = params.unix_group, + create_parents = True + ) + + Directory(params.ranger_pid_dir, + mode=0750, + create_parents=True, + owner = params.unix_user, + group = params.unix_group, + cd_access = "a", + ) + + Directory(params.tagsync_log_dir, + create_parents = True, + owner = params.unix_user, + group = params.unix_group, + cd_access = "a", + mode=0755 + ) + + File(format('{ranger_tagsync_conf}/ranger-tagsync-env-logdir.sh'), + content = format("export RANGER_TAGSYNC_LOG_DIR={tagsync_log_dir}"), + owner = params.unix_user, + group = params.unix_group, + mode=0755 + ) + + XmlConfig("ranger-tagsync-site.xml", + conf_dir=ranger_tagsync_conf, + configurations=params.config['configurations']['ranger-tagsync-site'], + configuration_attributes=params.config['configuration_attributes']['ranger-tagsync-site'], + owner=params.unix_user, + group=params.unix_group, + mode=0644) + + PropertiesFile(format('{ranger_tagsync_conf}/application.properties'), + properties = params.tagsync_application_properties, + mode=0755, + owner=params.unix_user, + group=params.unix_group + ) + + if upgrade_type is not None: + src_file = format('{ranger_tagsync_home}/ews/webapp/WEB-INF/classes/conf.dist/log4j.xml') + dst_file = format('{tagsync_log4j_file}') + Execute(('cp', '-f', src_file, dst_file), sudo=True) + + if os.path.isfile(tagsync_log4j_file): + File(tagsync_log4j_file, owner=params.unix_user, group=params.unix_group) + else: + Logger.warning('Required file {0} does not exist, copying the file to {1} path'.format(tagsync_log4j_file, ranger_tagsync_conf)) + src_file = format('{ranger_tagsync_home}/ews/webapp/WEB-INF/classes/conf.dist/log4j.xml') + dst_file = format('{tagsync_log4j_file}') + Execute(('cp', '-f', src_file, dst_file), sudo=True) + File(tagsync_log4j_file, owner=params.unix_user, group=params.unix_group) + + cred_file = format('{ranger_home}/ranger_credential_helper.py') + if os.path.isfile(format('{ranger_tagsync_home}/ranger_credential_helper.py')): + cred_file = format('{ranger_tagsync_home}/ranger_credential_helper.py') + + cred_lib = os.path.join(ranger_tagsync_home,"lib","*") + cred_setup_prefix = (cred_file, '-l', cred_lib) + + if not is_empty(params.tagsync_jceks_path) and not is_empty(params.ranger_tagsync_tagadmin_password) and params.tagsync_enabled: + cred_setup = cred_setup_prefix + ('-f', params.tagsync_jceks_path, '-k', 'tagadmin.user.password', '-v', PasswordString(params.ranger_tagsync_tagadmin_password), '-c', '1') + Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) + + File(params.tagsync_jceks_path, + owner = params.unix_user, + group = params.unix_group, + mode = 0640 + ) + + if os.path.isfile(tagsync_services_file): + File(tagsync_services_file, + mode = 0755, + ) + + Execute(('ln','-sf', format('{tagsync_services_file}'),'/usr/bin/ranger-tagsync'), + not_if=format("ls /usr/bin/ranger-tagsync"), + only_if=format("ls {tagsync_services_file}"), + sudo=True) http://git-wip-us.apache.org/repos/asf/ambari/blob/bf04d16b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/status_params.py index b932f88..251ca63 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/status_params.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/status_params.py @@ -24,4 +24,6 @@ from resource_management.libraries.functions.format import format config = Script.get_config() tmp_dir = Script.get_tmp_dir() -upgrade_marker_file = format("{tmp_dir}/rangeradmin_ru.inprogress") \ No newline at end of file +upgrade_marker_file = format("{tmp_dir}/rangeradmin_ru.inprogress") +ranger_pid_dir = config['configurations']['ranger-env']['ranger_pid_dir'] +tagsync_pid_file = format('{ranger_pid_dir}/tagsync.pid') \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/bf04d16b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json index 7a3c12e..0388aa2 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json @@ -42,6 +42,11 @@ "min_version": "2.2.0.0" }, { + "name": "ranger_tagsync_component", + "description": "Ranger Tagsync component support (AMBARI-14383)", + "min_version": "2.5.0.0" + }, + { "name": "phoenix", "description": "Phoenix Service support", "min_version": "2.3.0.0" http://git-wip-us.apache.org/repos/asf/ambari/blob/bf04d16b/ambari-server/src/main/resources/stacks/HDP/2.2/services/RANGER/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/RANGER/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/RANGER/metainfo.xml index d2bc6f0..0529aa6 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/RANGER/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/RANGER/metainfo.xml @@ -40,6 +40,10 @@ <package> <name>ranger_${stack_version}-usersync</name> </package> + <package> + <name>ranger_${stack_version}-tagsync</name> + <condition>should_install_ranger_tagsync</condition> + </package> </packages> </osSpecific> <osSpecific> @@ -51,6 +55,10 @@ <package> <name>ranger-${stack_version}-usersync</name> </package> + <package> + <name>ranger-${stack_version}-tagsync</name> + <condition>should_install_ranger_tagsync</condition> + </package> </packages> </osSpecific> </osSpecifics> http://git-wip-us.apache.org/repos/asf/ambari/blob/bf04d16b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/ranger-tagsync-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/ranger-tagsync-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/ranger-tagsync-site.xml new file mode 100644 index 0000000..cfcab3b --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/ranger-tagsync-site.xml @@ -0,0 +1,187 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration supports_final="true"> + + <property> + <name>ranger.tagsync.enabled</name> + <display-name>Enable Tagsync</display-name> + <value>true</value> + <description>Enable/Disable Ranger Tagsync</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + <type>value-list</type> + <overridable>false</overridable> + <entries> + <entry> + <value>true</value> + <label>Yes</label> + </entry> + <entry> + <value>false</value> + <label>No</label> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + </property> + + <property> + <name>ranger.tagsync.logdir</name> + <value>/var/log/ranger/tagsync</value> + <description>Ranger Log dir</description> + </property> + + <property> + <name>ranger.tagsync.tagadmin.rest.url</name> + <value>{{ranger_external_url}}</value> + <description>Ranger TagAdmin REST URL</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + + <property> + <name>ranger.tagsync.filesource.modtime.check.interval</name> + <display-name>File Source: File update polling interval</display-name> + <value></value> + <description></description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + + <property> + <name>ranger.tagsync.atlasrestsource.download.interval</name> + <display-name>AtlasREST Source: Atlas source download interval</display-name> + <value></value> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + + <property> + <name>ranger.tagsync.tagadmin.rest.ssl.config.file</name> + <value>/usr/hdp/current/ranger-tagsync/conf/mytruststore.jks</value> + <description>Truststore used for tagsync, required if tagsync -> ranger admin communication is SSL enabled</description> + </property> + + <property> + <name>ranger.tagsync.filesource.filename</name> + <display-name>File Source: Filename</display-name> + <value></value> + <description>File Source Filename</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + + <property> + <name>ranger.tagsync.source.impl.class</name> + <display-name>Tag Source</display-name> + <value>atlas</value> + <description>Tagsync Source IMPL class</description> + <value-attributes> + <type>value-list</type> + <empty-value-valid>true</empty-value-valid> + <overridable>false</overridable> + <entries> + <entry> + <value>atlas</value> + <label>ATLAS</label> + </entry> + <entry> + <value>file</value> + <label>FILE</label> + </entry> + <entry> + <value>atlasrest</value> + <label>ATLASREST</label> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + </property> + + <property> + <name>ranger.tagsync.tagadmin.keystore</name> + <value>/usr/hdp/current/ranger-tagsync/conf/rangertagsync.jceks</value> + <description>Keystore file</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + + <property> + <name>ranger.tagsync.tagadmin.alias</name> + <value>tagsync.tagadmin</value> + <description></description> + </property> + + <property> + <name>ranger.tagsync.tagadmin.password</name> + <display-name>Tagsync admin password</display-name> + <value></value> + <description>Tagadmin password</description> + <value-attributes> + <type>password</type> + </value-attributes> + </property> + + <property> + <name>ranger.tagsync.atlas.to.service.mapping</name> + <display-name>Tagsync Atlas to Ranger service mapping</display-name> + <value></value> + <description>Service Mapping</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + + <property> + <name>ranger.tagsync.atlasrestsource.endpoint</name> + <display-name>AtlasREST Source: Atlas endpoint</display-name> + <value></value> + <description></description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + + <property> + <name>ranger.tagsync.source.atlas.custom.resource.mappers</name> + <value></value> + <description></description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + + <property> + <name>ranger.tagsync.sink.impl.class</name> + <value>org.apache.ranger.tagsync.sink.tagadmin.TagAdminRESTSink</value> + <description></description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/bf04d16b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/tagsync-application-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/tagsync-application-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/tagsync-application-properties.xml new file mode 100644 index 0000000..d369098 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/tagsync-application-properties.xml @@ -0,0 +1,95 @@ +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + + <property> + <name>atlas.kafka.entities.group.id</name> + <display-name>Atlas Source: Kafka consumer group</display-name> + <value>ranger_entities_consumer</value> + <description></description> + </property> + + <property> + <name>atlas.kafka.bootstrap.servers</name> + <display-name>Atlas Source: Kafka endpoint</display-name> + <value>localhost:6667</value> + <description></description> + <depends-on> + <property> + <type>kafka-broker</type> + <name>port</name> + </property> + <property> + <type>ranger-tagsync-site</type> + <name>ranger.tagsync.enabled</name> + </property> + </depends-on> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + + <property> + <name>atlas.kafka.zookeeper.connect</name> + <display-name>Atlas Source: Zookeeper endpoint</display-name> + <value>localhost:2181</value> + <description></description> + <depends-on> + <property> + <type>zoo.cfg</type> + <name>clientPort</name> + </property> + <property> + <type>ranger-tagsync-site</type> + <name>ranger.tagsync.enabled</name> + </property> + </depends-on> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + + <property> + <name>atlas.notification.embedded</name> + <value>false</value> + <description></description> + </property> + + <property> + <name>atlas.kafka.acks</name> + <value>1</value> + <description></description> + </property> + + <property> + <name>atlas.kafka.data</name> + <value>/usr/hdp/current/atlas-server/data/kafka</value> + <description></description> + </property> + + <property> + <name>atlas.kafka.hook.group.id</name> + <value>atlas</value> + <description></description> + </property> + +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/bf04d16b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/metainfo.xml index 73a3788..9f96308 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/metainfo.xml @@ -23,7 +23,35 @@ <services> <service> <name>RANGER</name> - <version>0.5.0.2.5</version> + <version>0.6.0.2.5</version> + + <components> + <component> + <name>RANGER_TAGSYNC</name> + <displayName>Ranger Tagsync</displayName> + <category>MASTER</category> + <cardinality>1</cardinality> + <versionAdvertised>true</versionAdvertised> + <commandScript> + <script>scripts/ranger_tagsync.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + </components> + + <themes> + <theme> + <fileName>theme_version_3.json</fileName> + <default>true</default> + </theme> + </themes> + + <configuration-dependencies> + <config-type>ranger-tagsync-site</config-type> + <config-type>tagsync-application-properties</config-type> + </configuration-dependencies> + </service> </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/bf04d16b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/themes/theme_version_3.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/themes/theme_version_3.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/themes/theme_version_3.json new file mode 100644 index 0000000..19411d4 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/themes/theme_version_3.json @@ -0,0 +1,308 @@ +{ + "configuration": { + "layouts": [ + { + "name": "default", + "tabs": [ + { + "name": "ranger_tagsync", + "display-name": "Ranger Tagsync", + "layout": { + "tab-columns": "1", + "tab-rows": "1", + "sections": [ + { + "name": "section-tagsync", + "display-name": "Ranger Tagsync", + "row-index": "0", + "column-index": "0", + "row-span": "2", + "column-span": "1", + "section-columns": "1", + "section-rows": "2", + "subsections": [ + { + "name": "subsection-ranger-tagsync-row1-col1", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + }, + { + "name": "subsection-ranger-tagsync-row2-col1", + "row-index": "1", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "depends-on": [ + { + "configs": [ + "ranger-tagsync-site/ranger.tagsync.enabled" + ], + "if": "${ranger-tagsync-site/ranger.tagsync.enabled}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + } + ] + } + ] + } + } + ] + } + ], + "placement": { + "configuration-layout": "default", + "configs": [ + { + "config": "ranger-tagsync-site/ranger.tagsync.enabled", + "subsection-name": "subsection-ranger-tagsync-row1-col1" + }, + { + "config": "ranger-tagsync-site/ranger.tagsync.tagadmin.password", + "subsection-name": "subsection-ranger-tagsync-row2-col1" + }, + { + "config": "ranger-tagsync-site/ranger.tagsync.source.impl.class", + "subsection-name": "subsection-ranger-tagsync-row2-col1" + }, + { + "config": "ranger-tagsync-site/ranger.tagsync.filesource.modtime.check.interval", + "subsection-name": "subsection-ranger-tagsync-row2-col1", + "depends-on": [ + { + "configs":[ + "ranger-tagsync-site/ranger.tagsync.source.impl.class" + ], + "if": "${ranger-tagsync-site/ranger.tagsync.source.impl.class} === file", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "ranger-tagsync-site/ranger.tagsync.filesource.filename", + "subsection-name": "subsection-ranger-tagsync-row2-col1", + "depends-on": [ + { + "configs":[ + "ranger-tagsync-site/ranger.tagsync.source.impl.class" + ], + "if": "${ranger-tagsync-site/ranger.tagsync.source.impl.class} === file", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "ranger-tagsync-site/ranger.tagsync.atlasrestsource.endpoint", + "subsection-name": "subsection-ranger-tagsync-row2-col1", + "depends-on": [ + { + "configs":[ + "ranger-tagsync-site/ranger.tagsync.source.impl.class" + ], + "if": "${ranger-tagsync-site/ranger.tagsync.source.impl.class} === atlasrest", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "ranger-tagsync-site/ranger.tagsync.atlasrestsource.download.interval", + "subsection-name": "subsection-ranger-tagsync-row2-col1", + "depends-on": [ + { + "configs":[ + "ranger-tagsync-site/ranger.tagsync.source.impl.class" + ], + "if": "${ranger-tagsync-site/ranger.tagsync.source.impl.class} === atlasrest", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "ranger-tagsync-site/ranger.tagsync.atlas.to.service.mapping", + "subsection-name": "subsection-ranger-tagsync-row2-col1" + }, + { + "config": "tagsync-application-properties/atlas.kafka.entities.group.id", + "subsection-name": "subsection-ranger-tagsync-row2-col1", + "depends-on": [ + { + "configs":[ + "ranger-tagsync-site/ranger.tagsync.source.impl.class" + ], + "if": "${ranger-tagsync-site/ranger.tagsync.source.impl.class} === atlas", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "tagsync-application-properties/atlas.kafka.bootstrap.servers", + "subsection-name": "subsection-ranger-tagsync-row2-col1", + "depends-on": [ + { + "configs":[ + "ranger-tagsync-site/ranger.tagsync.source.impl.class" + ], + "if": "${ranger-tagsync-site/ranger.tagsync.source.impl.class} === atlas", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "tagsync-application-properties/atlas.kafka.zookeeper.connect", + "subsection-name": "subsection-ranger-tagsync-row2-col1", + "depends-on": [ + { + "configs":[ + "ranger-tagsync-site/ranger.tagsync.source.impl.class" + ], + "if": "${ranger-tagsync-site/ranger.tagsync.source.impl.class} === atlas", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + } + ] + }, + "widgets": [ + { + "config": "ranger-tagsync-site/ranger.tagsync.enabled", + "widget": { + "type": "toggle" + } + }, + { + "config": "ranger-tagsync-site/ranger.tagsync.tagadmin.password", + "widget": { + "type": "password" + } + }, + { + "config": "ranger-tagsync-site/ranger.tagsync.source.impl.class", + "widget": { + "type": "combo" + } + }, + { + "config": "ranger-tagsync-site/ranger.tagsync.atlas.to.service.mapping", + "widget": { + "type": "text-field" + } + }, + { + "config": "ranger-tagsync-site/ranger.tagsync.filesource.modtime.check.interval", + "widget": { + "type": "text-field" + } + }, + { + "config": "ranger-tagsync-site/ranger.tagsync.filesource.filename", + "widget": { + "type": "text-field" + } + }, + { + "config": "ranger-tagsync-site/ranger.tagsync.atlasrestsource.download.interval", + "widget": { + "type": "text-field" + } + }, + { + "config": "ranger-tagsync-site/ranger.tagsync.atlasrestsource.endpoint", + "widget": { + "type": "text-field" + } + }, + { + "config": "tagsync-application-properties/atlas.kafka.entities.group.id", + "widget": { + "type": "text-field" + } + }, + { + "config": "tagsync-application-properties/atlas.kafka.bootstrap.servers", + "widget": { + "type": "text-field" + } + }, + { + "config": "tagsync-application-properties/atlas.kafka.zookeeper.connect", + "widget": { + "type": "text-field" + } + } + ] + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/bf04d16b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index 9dd5cc7..9050092 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -19,4 +19,38 @@ limitations under the License. class HDP25StackAdvisor(HDP24StackAdvisor): - pass + + def getServiceConfigurationRecommenderDict(self): + parentRecommendConfDict = super(HDP25StackAdvisor, self).getServiceConfigurationRecommenderDict() + childRecommendConfDict = { + "RANGER": self.recommendRangerConfigurations + } + parentRecommendConfDict.update(childRecommendConfDict) + return parentRecommendConfDict + + def recommendRangerConfigurations(self, configurations, clusterData, services, hosts): + super(HDP25StackAdvisor, self).recommendRangerConfigurations(configurations, clusterData, services, hosts) + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + + putTagsyncAppProperty = self.putProperty(configurations, "tagsync-application-properties", services) + + zookeeper_host_port = self.getZKHostPortString(services) + if zookeeper_host_port: + zookeeper_host_list = zookeeper_host_port.split(',') + putTagsyncAppProperty('atlas.kafka.zookeeper.connect', zookeeper_host_list[0]) + else: + putTagsyncAppProperty('atlas.kafka.zookeeper.connect', 'localhost:6667') + + if 'KAFKA' in servicesList: + kafka_hosts = self.getHostNamesWithComponent("KAFKA", "KAFKA_BROKER", services) + kafka_port = '6667' + if 'kafka-broker' in services['configurations'] and ('port' in services['configurations']['kafka-broker']['properties']): + kafka_port = services['configurations']['kafka-broker']['properties']['port'] + kafka_host_port = [] + for i in range(len(kafka_hosts)): + kafka_host_port.append(kafka_hosts[i] + ':' + kafka_port) + + final_kafka_host = ",".join(kafka_host_port) + putTagsyncAppProperty('atlas.kafka.bootstrap.servers', final_kafka_host) + else: + putTagsyncAppProperty('atlas.kafka.bootstrap.servers', 'localhost:2181') http://git-wip-us.apache.org/repos/asf/ambari/blob/bf04d16b/ambari-server/src/main/resources/stacks/HDP/2.6/services/RANGER/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/RANGER/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/RANGER/metainfo.xml index 18c3333..8f0f715 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/RANGER/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/RANGER/metainfo.xml @@ -23,7 +23,7 @@ <services> <service> <name>RANGER</name> - <version>0.5.0.2.6</version> + <version>0.6.0.2.6</version> </service> </services> </metainfo>
