AMBARI-19044 Install & configure Ranger plugin components independently of Ranger admin components (mugdha)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1524fd77 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1524fd77 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1524fd77 Branch: refs/heads/branch-dev-patch-upgrade Commit: 1524fd775d4b25d0896c648cb1bbc8ed3644a73d Parents: 8b22dd0 Author: Mugdha Varadkar <[email protected]> Authored: Tue Jan 17 17:08:02 2017 +0530 Committer: Mugdha Varadkar <[email protected]> Committed: Tue Jan 17 18:16:38 2017 +0530 ---------------------------------------------------------------------- .../libraries/functions/constants.py | 3 + .../functions/setup_ranger_plugin_xml.py | 47 +++- .../server/upgrade/UpgradeCatalog250.java | 37 ++++ .../ATLAS/0.1.0.2.3/package/scripts/params.py | 71 ++++-- .../package/scripts/setup_ranger_atlas.py | 4 +- .../0.96.0.2.0/package/scripts/params_linux.py | 163 +++++++------- .../package/scripts/setup_ranger_hbase.py | 4 +- .../2.1.0.2.0/package/scripts/params_linux.py | 166 +++++++------- .../package/scripts/setup_ranger_hdfs.py | 44 ++-- .../0.12.0.2.0/package/scripts/params_linux.py | 161 +++++++------- .../package/scripts/setup_ranger_hive.py | 6 +- .../KAFKA/0.8.1/package/scripts/params.py | 126 +++++------ .../0.8.1/package/scripts/setup_ranger_kafka.py | 4 +- .../0.9.0/configuration/ranger-kafka-audit.xml | 32 +-- .../ranger-kafka-plugin-properties.xml | 14 +- .../ranger-kafka-policymgr-ssl.xml | 12 +- .../configuration/ranger-kafka-security.xml | 18 +- .../ranger-knox-plugin-properties.xml | 12 +- .../0.5.0.2.2/package/scripts/params_linux.py | 155 ++++++------- .../package/scripts/setup_ranger_knox.py | 5 +- .../configuration/ranger-kms-security.xml | 6 + .../0.10.0/configuration/ranger-storm-audit.xml | 32 +-- .../ranger-storm-policymgr-ssl.xml | 12 +- .../configuration/ranger-storm-security.xml | 18 +- .../STORM/0.9.1/package/scripts/params_linux.py | 161 +++++++------- .../0.9.1/package/scripts/setup_ranger_storm.py | 4 +- .../ranger-storm-plugin-properties.xml | 71 ++++++ .../2.1.0.2.0/package/scripts/params_linux.py | 215 ++++++++++--------- .../package/scripts/resourcemanager.py | 2 +- .../package/scripts/setup_ranger_yarn.py | 4 +- .../HDP/2.0.6/properties/stack_features.json | 17 +- .../ranger-hbase-plugin-properties.xml | 10 +- .../ranger-hdfs-plugin-properties.xml | 12 +- .../ranger-hive-plugin-properties.xml | 10 +- .../ranger-knox-plugin-properties.xml | 2 +- .../stacks/HDP/2.2/services/stack_advisor.py | 38 ++-- .../HBASE/configuration/ranger-hbase-audit.xml | 32 +-- .../ranger-hbase-policymgr-ssl.xml | 12 +- .../configuration/ranger-hbase-security.xml | 20 +- .../configuration/ranger-hdfs-policymgr-ssl.xml | 12 +- .../HDFS/configuration/ranger-hdfs-security.xml | 20 +- .../HIVE/configuration/ranger-hive-audit.xml | 32 +-- .../configuration/ranger-hive-policymgr-ssl.xml | 12 +- .../HIVE/configuration/ranger-hive-security.xml | 20 +- .../ranger-kafka-policymgr-ssl.xml | 4 +- .../KNOX/configuration/ranger-knox-audit.xml | 32 +-- .../configuration/ranger-knox-policymgr-ssl.xml | 12 +- .../KNOX/configuration/ranger-knox-security.xml | 18 +- .../ranger-storm-policymgr-ssl.xml | 4 +- .../configuration/ranger-storm-security.xml | 2 +- .../YARN/configuration/ranger-yarn-audit.xml | 32 +-- .../ranger-yarn-plugin-properties.xml | 12 +- .../configuration/ranger-yarn-policymgr-ssl.xml | 12 +- .../YARN/configuration/ranger-yarn-security.xml | 18 +- .../stacks/HDP/2.3/services/stack_advisor.py | 34 +++ .../ATLAS/configuration/ranger-atlas-audit.xml | 6 +- .../ranger-atlas-plugin-properties.xml | 58 ++++- .../ranger-atlas-policymgr-ssl.xml | 12 +- .../configuration/ranger-atlas-security.xml | 20 +- .../ranger-hbase-plugin-properties.xml | 71 ++++++ .../ranger-hdfs-plugin-properties.xml | 50 ++++- .../ranger-hive-plugin-properties.xml | 71 ++++++ .../HIVE/configuration/ranger-hive-security.xml | 2 +- .../ranger-kafka-plugin-properties.xml | 71 ++++++ .../ranger-knox-plugin-properties.xml | 71 ++++++ .../ranger-storm-policymgr-ssl.xml | 4 +- .../configuration/ranger-storm-security.xml | 2 +- .../ranger-yarn-plugin-properties.xml | 71 ++++++ .../stacks/HDP/2.5/services/stack_advisor.py | 7 + .../server/upgrade/UpgradeCatalog250Test.java | 110 ++++++++++ .../stacks/2.0.6/configs/altfs_plus_hdfs.json | 6 +- .../python/stacks/2.0.6/configs/default.json | 10 +- .../stacks/2.0.6/configs/default_client.json | 3 +- .../2.0.6/configs/default_hive_nn_ha.json | 3 +- .../2.0.6/configs/default_hive_nn_ha_2.json | 3 +- .../2.0.6/configs/default_hive_non_hdfs.json | 3 +- .../2.0.6/configs/default_no_install.json | 3 +- .../2.0.6/configs/default_with_bucket.json | 4 +- .../2.0.6/configs/ha_bootstrap_active_node.json | 2 +- .../configs/ha_bootstrap_standby_node.json | 2 +- ...ha_bootstrap_standby_node_initial_start.json | 2 +- ...dby_node_initial_start_dfs_nameservices.json | 2 +- .../python/stacks/2.0.6/configs/ha_default.json | 4 +- .../python/stacks/2.0.6/configs/ha_secured.json | 2 +- .../python/stacks/2.0.6/configs/hbase-2.2.json | 4 +- .../2.0.6/configs/hbase-rs-2.2-phoenix.json | 4 +- .../stacks/2.0.6/configs/hbase-rs-2.2.json | 4 +- .../python/stacks/2.0.6/configs/nn_ru_lzo.json | 2 +- .../python/stacks/2.0.6/configs/secured.json | 12 +- .../stacks/2.0.6/configs/secured_client.json | 3 +- .../stacks/2.1/configs/default-storm-start.json | 2 +- .../test/python/stacks/2.1/configs/default.json | 5 +- .../stacks/2.1/configs/secured-storm-start.json | 2 +- .../test/python/stacks/2.1/configs/secured.json | 5 +- .../stacks/2.2/common/test_stack_advisor.py | 53 ++++- .../test/python/stacks/2.2/configs/default.json | 6 +- .../python/stacks/2.2/configs/hive-upgrade.json | 3 +- .../stacks/2.3/common/test_stack_advisor.py | 3 +- .../python/stacks/2.5/configs/hsi_default.json | 3 +- .../test/python/stacks/2.5/configs/hsi_ha.json | 3 +- .../controllers/main/service/info/configs.js | 4 +- .../app/controllers/wizard/step7_controller.js | 4 +- 102 files changed, 1889 insertions(+), 946 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-common/src/main/python/resource_management/libraries/functions/constants.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py index 56af615..6895e34 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py @@ -106,6 +106,9 @@ class StackFeature: ZKFC_VERSION_ADVERTISED = "zkfc_version_advertised" PHOENIX_CORE_HDFS_SITE_REQUIRED = "phoenix_core_hdfs_site_required" RANGER_TAGSYNC_SSL_XML_SUPPORT="ranger_tagsync_ssl_xml_support" + RANGER_XML_CONFIGURATION = "ranger_xml_configuration" + KAFKA_RANGER_PLUGIN_SUPPORT = "kafka_ranger_plugin_support" + YARN_RANGER_PLUGIN_SUPPORT = "yarn_ranger_plugin_support" RANGER_SOLR_CONFIG_SUPPORT='ranger_solr_config_support' HIVE_INTERACTIVE_ATLAS_HOOK_REQUIRED="hive_interactive_atlas_hook_required" CORE_SITE_FOR_RANGER_PLUGINS_SUPPORT='core_site_for_ranger_plugins' http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py index 6561928..a12116d 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py @@ -17,8 +17,7 @@ See the License for the specific language governing permissions and limitations under the License. """ -__all__ = ["setup_ranger_plugin"] - +__all__ = ["setup_ranger_plugin", "get_audit_configs"] import os import ambari_simplejson as json @@ -34,6 +33,7 @@ from resource_management.libraries.functions.ranger_functions_v2 import Rangerad from resource_management.core.utils import PasswordString from resource_management.libraries.script.script import Script from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.default import default def setup_ranger_plugin(component_select_name, service_name, previous_jdbc_jar, component_downloaded_custom_connector, component_driver_curl_source, @@ -164,8 +164,8 @@ def setup_ranger_plugin(component_select_name, service_name, previous_jdbc_jar, group = component_group, mode=0744) - #This should be done by rpm - #setup_ranger_plugin_jar_symblink(stack_version, service_name, component_list) + # creating symblink should be done by rpm package + # setup_ranger_plugin_jar_symblink(stack_version, service_name, component_list) setup_ranger_plugin_keystore(service_name, audit_db_is_enabled, stack_version, credential_file, xa_audit_db_password, ssl_truststore_password, ssl_keystore_password, @@ -176,7 +176,6 @@ def setup_ranger_plugin(component_select_name, service_name, previous_jdbc_jar, action="delete" ) - def setup_ranger_plugin_jar_symblink(stack_version, service_name, component_list): stack_root = Script.get_stack_root() @@ -217,7 +216,6 @@ def setup_ranger_plugin_keystore(service_name, audit_db_is_enabled, stack_versio mode = 0640 ) - def setup_core_site_for_required_plugins(component_user, component_group, create_core_site_path, config): XmlConfig('core-site.xml', conf_dir=create_core_site_path, @@ -227,3 +225,40 @@ def setup_core_site_for_required_plugins(component_user, component_group, create group=component_group, mode=0644 ) + +def get_audit_configs(config): + xa_audit_db_flavor = config['configurations']['admin-properties']['DB_FLAVOR'].lower() + xa_db_host = config['configurations']['admin-properties']['db_host'] + xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits') + + if xa_audit_db_flavor == 'mysql': + jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None) + audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}') + jdbc_driver = "com.mysql.jdbc.Driver" + elif xa_audit_db_flavor == 'oracle': + jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None) + colon_count = xa_db_host.count(':') + if colon_count == 2 or colon_count == 0: + audit_jdbc_url = format('jdbc:oracle:thin:@{xa_db_host}') + else: + audit_jdbc_url = format('jdbc:oracle:thin:@//{xa_db_host}') + jdbc_driver = "oracle.jdbc.OracleDriver" + elif xa_audit_db_flavor == 'postgres': + jdbc_jar_name = default("/hostLevelParams/custom_postgres_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None) + audit_jdbc_url = format('jdbc:postgresql://{xa_db_host}/{xa_audit_db_name}') + jdbc_driver = "org.postgresql.Driver" + elif xa_audit_db_flavor == 'mssql': + jdbc_jar_name = default("/hostLevelParams/custom_mssql_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None) + audit_jdbc_url = format('jdbc:sqlserver://{xa_db_host};databaseName={xa_audit_db_name}') + jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver" + elif xa_audit_db_flavor == 'sqla': + jdbc_jar_name = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None) + audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}') + jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver" + + return jdbc_jar_name, previous_jdbc_jar_name, audit_jdbc_url, jdbc_driver \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java index 29e1f17..6638379 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java @@ -166,6 +166,7 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { updateLogSearchConfigs(); updateAmbariInfraConfigs(); updateYarnSite(); + updateRangerUrlConfigs(); addManageServiceAutoStartPermissions(); } @@ -881,4 +882,40 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { "CLUSTER.OPERATOR:CLUSTER"); addRoleAuthorization("CLUSTER.MANAGE_AUTO_START", "Manage service auto-start configuration", roles); } + + /** + * Updates Ranger admin url for Ranger plugin supported configs. + * + * @throws AmbariException + */ + protected void updateRangerUrlConfigs() throws AmbariException { + AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); + for (final Cluster cluster : getCheckedClusterMap(ambariManagementController.getClusters()).values()) { + + Config ranger_admin_properties = cluster.getDesiredConfigByType("admin-properties"); + if(null != ranger_admin_properties) { + String policyUrl = ranger_admin_properties.getProperties().get("policymgr_external_url"); + if (null != policyUrl) { + updateRangerUrl(cluster, "ranger-hdfs-security", "ranger.plugin.hdfs.policy.rest.url", policyUrl); + updateRangerUrl(cluster, "ranger-hive-security", "ranger.plugin.hive.policy.rest.url", policyUrl); + updateRangerUrl(cluster, "ranger-hbase-security", "ranger.plugin.hbase.policy.rest.url", policyUrl); + updateRangerUrl(cluster, "ranger-knox-security", "ranger.plugin.knox.policy.rest.url", policyUrl); + updateRangerUrl(cluster, "ranger-storm-security", "ranger.plugin.storm.policy.rest.url", policyUrl); + updateRangerUrl(cluster, "ranger-yarn-security", "ranger.plugin.yarn.policy.rest.url", policyUrl); + updateRangerUrl(cluster, "ranger-kafka-security", "ranger.plugin.kafka.policy.rest.url", policyUrl); + updateRangerUrl(cluster, "ranger-atlas-security", "ranger.plugin.atlas.policy.rest.url", policyUrl); + updateRangerUrl(cluster, "ranger-kms-security", "ranger.plugin.kms.policy.rest.url", policyUrl); + } + } + } + } + + protected void updateRangerUrl(Cluster cluster, String configType, String configProperty, String policyUrl) throws AmbariException { + Config componentSecurity = cluster.getDesiredConfigByType(configType); + if(componentSecurity != null && componentSecurity.getProperties().containsKey(configProperty)) { + Map<String, String> updateProperty = new HashMap<>(); + updateProperty.put(configProperty, policyUrl); + updateConfigurationPropertiesForCluster(cluster, configType, updateProperty, true, false); + } + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py index 94193be..c74d046 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py @@ -219,17 +219,7 @@ for host in zookeeper_hosts: if index < len(zookeeper_hosts): zookeeper_quorum += "," - -# Atlas Ranger plugin configurations -stack_supports_atlas_ranger_plugin = check_stack_feature(StackFeature.ATLAS_RANGER_PLUGIN_SUPPORT, version_for_stack_feature_checks) -stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks) stack_supports_atlas_hdfs_site_on_namenode_ha = check_stack_feature(StackFeature.ATLAS_HDFS_SITE_ON_NAMENODE_HA, version_for_stack_feature_checks) -retry_enabled = default("/commandParams/command_retry_enabled", False) - -ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) -has_ranger_admin = not len(ranger_admin_hosts) == 0 -xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported'] -enable_ranger_atlas = False atlas_server_xmx = default("configurations/atlas-env/atlas_server_xmx", 2048) atlas_server_max_new_size = default("configurations/atlas-env/atlas_server_max_new_size", 614) @@ -237,9 +227,6 @@ atlas_server_max_new_size = default("configurations/atlas-env/atlas_server_max_n hbase_master_hosts = default('/clusterHostInfo/hbase_master_hosts', []) has_hbase_master = not len(hbase_master_hosts) == 0 -ranger_admin_hosts = default('/clusterHostInfo/ranger_admin_hosts', []) -has_ranger_admin = not len(ranger_admin_hosts) == 0 - atlas_hbase_setup = format("{exec_tmp_dir}/atlas_hbase_setup.rb") atlas_kafka_setup = format("{exec_tmp_dir}/atlas_kafka_acl.sh") atlas_graph_storage_hbase_table = default('/configurations/application-properties/atlas.graph.storage.hbase.table', None) @@ -247,7 +234,6 @@ atlas_audit_hbase_tablename = default('/configurations/application-properties/at hbase_user_keytab = default('/configurations/hbase-env/hbase_user_keytab', None) hbase_principal_name = default('/configurations/hbase-env/hbase_principal_name', None) -enable_ranger_hbase = False # ToDo: Kafka port to Atlas # Used while upgrading the stack in a kerberized cluster and running kafka-acls.sh @@ -289,7 +275,29 @@ if check_stack_feature(StackFeature.ATLAS_UPGRADE_SUPPORT, version_for_stack_fea namenode_host = set(default("/clusterHostInfo/namenode_host", [])) has_namenode = not len(namenode_host) == 0 -if has_ranger_admin and stack_supports_atlas_ranger_plugin: +# ranger altas plugin section start + +# ranger host +ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) +has_ranger_admin = not len(ranger_admin_hosts) == 0 + +retry_enabled = default("/commandParams/command_retry_enabled", False) + +stack_supports_atlas_ranger_plugin = check_stack_feature(StackFeature.ATLAS_RANGER_PLUGIN_SUPPORT, version_for_stack_feature_checks) +stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks) + +# ranger support xml_configuration flag, instead of depending on ranger xml_configurations_supported/ranger-env, using stack feature +xml_configurations_supported = check_stack_feature(StackFeature.RANGER_XML_CONFIGURATION, version_for_stack_feature_checks) + +# ranger atlas plugin enabled property +enable_ranger_atlas = default("/configurations/ranger-atlas-plugin-properties/ranger-atlas-plugin-enabled", "No") +enable_ranger_atlas = True if enable_ranger_atlas.lower() == "yes" else False + +# ranger hbase plugin enabled property +enable_ranger_hbase = default("/configurations/ranger-hbase-plugin-properties/ranger-hbase-plugin-enabled", "No") +enable_ranger_hbase = True if enable_ranger_hbase.lower() == 'yes' else False + +if stack_supports_atlas_ranger_plugin and enable_ranger_atlas: # for create_hdfs_directory hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] if has_namenode else None hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] if has_namenode else None @@ -320,27 +328,42 @@ if has_ranger_admin and stack_supports_atlas_ranger_plugin: dfs_type = dfs_type ) + # ranger atlas service/repository name repo_name = str(config['clusterName']) + '_atlas' repo_name_value = config['configurations']['ranger-atlas-security']['ranger.plugin.atlas.service.name'] if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}": repo_name = repo_name_value - ssl_keystore_password = unicode(config['configurations']['ranger-atlas-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) - ssl_truststore_password = unicode(config['configurations']['ranger-atlas-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) + + ssl_keystore_password = config['configurations']['ranger-atlas-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password'] + ssl_truststore_password = config['configurations']['ranger-atlas-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password'] credential_file = format('/etc/ranger/{repo_name}/cred.jceks') xa_audit_hdfs_is_enabled = default('/configurations/ranger-atlas-audit/xasecure.audit.destination.hdfs', False) - enable_ranger_atlas = config['configurations']['ranger-atlas-plugin-properties']['ranger-atlas-plugin-enabled'] - enable_ranger_atlas = not is_empty(enable_ranger_atlas) and enable_ranger_atlas.lower() == 'yes' - enable_ranger_hbase = config['configurations']['ranger-hbase-plugin-properties']['ranger-hbase-plugin-enabled'] - enable_ranger_hbase = not is_empty(enable_ranger_hbase) and enable_ranger_hbase.lower() == 'yes' - policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url'] + + # get ranger policy url + policymgr_mgr_url = config['configurations']['ranger-atlas-security']['ranger.plugin.atlas.policy.rest.url'] + + if not is_empty(policymgr_mgr_url) and policymgr_mgr_url.endswith('/'): + policymgr_mgr_url = policymgr_mgr_url.rstrip('/') downloaded_custom_connector = None driver_curl_source = None driver_curl_target = None ranger_env = config['configurations']['ranger-env'] - ranger_plugin_properties = config['configurations']['ranger-atlas-plugin-properties'] + # create ranger-env config having external ranger credential properties + if not has_ranger_admin and enable_ranger_atlas: + external_admin_username = default('/configurations/ranger-atlas-plugin-properties/external_admin_username', 'admin') + external_admin_password = default('/configurations/ranger-atlas-plugin-properties/external_admin_password', 'admin') + external_ranger_admin_username = default('/configurations/ranger-atlas-plugin-properties/external_ranger_admin_username', 'amb_ranger_admin') + external_ranger_admin_password = default('/configurations/ranger-atlas-plugin-properties/external_ranger_admin_password', 'amb_ranger_admin') + ranger_env = {} + ranger_env['admin_username'] = external_admin_username + ranger_env['admin_password'] = external_admin_password + ranger_env['ranger_admin_username'] = external_ranger_admin_username + ranger_env['ranger_admin_password'] = external_ranger_admin_password + + ranger_plugin_properties = config['configurations']['ranger-atlas-plugin-properties'] ranger_atlas_audit = config['configurations']['ranger-atlas-audit'] ranger_atlas_audit_attrs = config['configuration_attributes']['ranger-atlas-audit'] ranger_atlas_security = config['configurations']['ranger-atlas-security'] @@ -357,6 +380,7 @@ if has_ranger_admin and stack_supports_atlas_ranger_plugin: 'commonNameForCertificate' : config['configurations']['ranger-atlas-plugin-properties']['common.name.for.certificate'], 'ambari.service.check.user' : policy_user } + if security_enabled: atlas_repository_configuration['policy.download.auth.users'] = metadata_user atlas_repository_configuration['tag.download.auth.users'] = metadata_user @@ -368,3 +392,4 @@ if has_ranger_admin and stack_supports_atlas_ranger_plugin: 'name': repo_name, 'type': 'atlas', } +# ranger atlas plugin section end http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/setup_ranger_atlas.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/setup_ranger_atlas.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/setup_ranger_atlas.py index f5d7f38..c47c75c 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/setup_ranger_atlas.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/setup_ranger_atlas.py @@ -19,7 +19,7 @@ from resource_management.core.logger import Logger def setup_ranger_atlas(upgrade_type=None): import params - if params.has_ranger_admin: + if params.enable_ranger_atlas: from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin @@ -67,4 +67,4 @@ def setup_ranger_atlas(upgrade_type=None): component_user_principal=params.atlas_jaas_principal if params.security_enabled else None, component_user_keytab=params.atlas_keytab_path if params.security_enabled else None) else: - Logger.info('Ranger admin not installed') \ No newline at end of file + Logger.info('Ranger Atlas plugin is not enabled') \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py index e27fd72..268d81c 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py @@ -41,6 +41,7 @@ from resource_management.libraries.functions.get_not_managed_resources import ge from resource_management.libraries.script.script import Script from resource_management.libraries.functions.expect import expect from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_from_hostnames +from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs # server configurations config = Script.get_config() @@ -232,8 +233,6 @@ hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] - - hdfs_site = config['configurations']['hdfs-site'] default_fs = config['configurations']['core-site']['fs.defaultFS'] @@ -258,87 +257,90 @@ HdfsResource = functools.partial( dfs_type = dfs_type ) -# ranger host -ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) -has_ranger_admin = not len(ranger_admin_hosts) == 0 -xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported'] -ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] - -# ranger hbase properties -policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url'] -if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'): - policymgr_mgr_url = policymgr_mgr_url.rstrip('/') -xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits') -xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger') -xa_db_host = config['configurations']['admin-properties']['db_host'] -repo_name = str(config['clusterName']) + '_hbase' -repo_name_value = config['configurations']['ranger-hbase-security']['ranger.plugin.hbase.service.name'] -if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}": - repo_name = repo_name_value - -common_name_for_certificate = config['configurations']['ranger-hbase-plugin-properties']['common.name.for.certificate'] - zookeeper_znode_parent = config['configurations']['hbase-site']['zookeeper.znode.parent'] hbase_zookeeper_quorum = config['configurations']['hbase-site']['hbase.zookeeper.quorum'] hbase_zookeeper_property_clientPort = config['configurations']['hbase-site']['hbase.zookeeper.property.clientPort'] hbase_security_authentication = config['configurations']['hbase-site']['hbase.security.authentication'] hadoop_security_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -repo_config_username = config['configurations']['ranger-hbase-plugin-properties']['REPOSITORY_CONFIG_USERNAME'] +# ranger hbase plugin section start -ranger_env = config['configurations']['ranger-env'] -ranger_plugin_properties = config['configurations']['ranger-hbase-plugin-properties'] -policy_user = config['configurations']['ranger-hbase-plugin-properties']['policy_user'] - -#For curl command in ranger plugin to get db connector +# to get db connector jar jdk_location = config['hostLevelParams']['jdk_location'] -java_share_dir = '/usr/share/java' -enable_ranger_hbase = False -if has_ranger_admin: - enable_ranger_hbase = (config['configurations']['ranger-hbase-plugin-properties']['ranger-hbase-plugin-enabled'].lower() == 'yes') + +# ranger host +ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) +has_ranger_admin = not len(ranger_admin_hosts) == 0 + +# ranger support xml_configuration flag, instead of depending on ranger xml_configurations_supported/ranger-env introduced, using stack feature +xml_configurations_supported = check_stack_feature(StackFeature.RANGER_XML_CONFIGURATION, version_for_stack_feature_checks) + +# ambari-server hostname +ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] + +# ranger hbase plugin enabled property +enable_ranger_hbase = default("/configurations/ranger-hbase-plugin-properties/ranger-hbase-plugin-enabled", "No") +enable_ranger_hbase = True if enable_ranger_hbase.lower() == 'yes' else False + +# ranger hbase properties +if enable_ranger_hbase: + # get ranger policy url + policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url'] + if xml_configurations_supported: + policymgr_mgr_url = config['configurations']['ranger-hbase-security']['ranger.plugin.hbase.policy.rest.url'] + + if not is_empty(policymgr_mgr_url) and policymgr_mgr_url.endswith('/'): + policymgr_mgr_url = policymgr_mgr_url.rstrip('/') + + # ranger audit db user + xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger') + + # ranger hbase service/repository name + repo_name = str(config['clusterName']) + '_hbase' + repo_name_value = config['configurations']['ranger-hbase-security']['ranger.plugin.hbase.service.name'] + if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}": + repo_name = repo_name_value + + common_name_for_certificate = config['configurations']['ranger-hbase-plugin-properties']['common.name.for.certificate'] + repo_config_username = config['configurations']['ranger-hbase-plugin-properties']['REPOSITORY_CONFIG_USERNAME'] + ranger_plugin_properties = config['configurations']['ranger-hbase-plugin-properties'] + policy_user = config['configurations']['ranger-hbase-plugin-properties']['policy_user'] + repo_config_password = config['configurations']['ranger-hbase-plugin-properties']['REPOSITORY_CONFIG_PASSWORD'] + + # ranger-env config + ranger_env = config['configurations']['ranger-env'] + + # create ranger-env config having external ranger credential properties + if not has_ranger_admin and enable_ranger_hbase: + external_admin_username = default('/configurations/ranger-hbase-plugin-properties/external_admin_username', 'admin') + external_admin_password = default('/configurations/ranger-hbase-plugin-properties/external_admin_password', 'admin') + external_ranger_admin_username = default('/configurations/ranger-hbase-plugin-properties/external_ranger_admin_username', 'amb_ranger_admin') + external_ranger_admin_password = default('/configurations/ranger-hbase-plugin-properties/external_ranger_admin_password', 'amb_ranger_admin') + ranger_env = {} + ranger_env['admin_username'] = external_admin_username + ranger_env['admin_password'] = external_admin_password + ranger_env['ranger_admin_username'] = external_ranger_admin_username + ranger_env['ranger_admin_password'] = external_ranger_admin_password + xa_audit_db_password = '' - if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db: - xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) - repo_config_password = unicode(config['configurations']['ranger-hbase-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']) - xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower() + if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db and has_ranger_admin: + xa_audit_db_password = config['configurations']['admin-properties']['audit_db_password'] + + downloaded_custom_connector = None previous_jdbc_jar_name = None + driver_curl_source = None + driver_curl_target = None + previous_jdbc_jar = None + + if has_ranger_admin and stack_supports_ranger_audit_db: + xa_audit_db_flavor = config['configurations']['admin-properties']['DB_FLAVOR'] + jdbc_jar_name, previous_jdbc_jar_name, audit_jdbc_url, jdbc_driver = get_audit_configs(config) - if stack_supports_ranger_audit_db: - if xa_audit_db_flavor == 'mysql': - jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None) - previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None) - audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}') - jdbc_driver = "com.mysql.jdbc.Driver" - elif xa_audit_db_flavor == 'oracle': - jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name", None) - previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None) - colon_count = xa_db_host.count(':') - if colon_count == 2 or colon_count == 0: - audit_jdbc_url = format('jdbc:oracle:thin:@{xa_db_host}') - else: - audit_jdbc_url = format('jdbc:oracle:thin:@//{xa_db_host}') - jdbc_driver = "oracle.jdbc.OracleDriver" - elif xa_audit_db_flavor == 'postgres': - jdbc_jar_name = default("/hostLevelParams/custom_postgres_jdbc_name", None) - previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None) - audit_jdbc_url = format('jdbc:postgresql://{xa_db_host}/{xa_audit_db_name}') - jdbc_driver = "org.postgresql.Driver" - elif xa_audit_db_flavor == 'mssql': - jdbc_jar_name = default("/hostLevelParams/custom_mssql_jdbc_name", None) - previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None) - audit_jdbc_url = format('jdbc:sqlserver://{xa_db_host};databaseName={xa_audit_db_name}') - jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver" - elif xa_audit_db_flavor == 'sqla': - jdbc_jar_name = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None) - previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None) - audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}') - jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver" - - downloaded_custom_connector = format("{exec_tmp_dir}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None - driver_curl_source = format("{jdk_location}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None - driver_curl_target = format("{stack_root}/current/{component_directory}/lib/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None - previous_jdbc_jar = format("{stack_root}/current/{component_directory}/lib/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None - sql_connector_jar = '' + downloaded_custom_connector = format("{exec_tmp_dir}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None + driver_curl_source = format("{jdk_location}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None + driver_curl_target = format("{stack_root}/current/{component_directory}/lib/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None + previous_jdbc_jar = format("{stack_root}/current/{component_directory}/lib/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None + sql_connector_jar = '' if security_enabled: master_principal = config['configurations']['hbase-site']['hbase.master.kerberos.principal'] @@ -385,23 +387,24 @@ if has_ranger_admin: if stack_supports_ranger_kerberos and security_enabled and 'hbase-master' in component_directory.lower(): ranger_hbase_principal = master_jaas_princ ranger_hbase_keytab = master_keytab_path - elif stack_supports_ranger_kerberos and security_enabled and 'hbase-regionserver' in component_directory.lower(): + elif stack_supports_ranger_kerberos and security_enabled and 'hbase-regionserver' in component_directory.lower(): ranger_hbase_principal = regionserver_jaas_princ ranger_hbase_keytab = regionserver_keytab_path xa_audit_db_is_enabled = False - ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls'] if xml_configurations_supported and stack_supports_ranger_audit_db: xa_audit_db_is_enabled = config['configurations']['ranger-hbase-audit']['xasecure.audit.destination.db'] - xa_audit_hdfs_is_enabled = config['configurations']['ranger-hbase-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None - ssl_keystore_password = unicode(config['configurations']['ranger-hbase-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None - ssl_truststore_password = unicode(config['configurations']['ranger-hbase-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None - credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None - #For SQLA explicitly disable audit to DB for Ranger - if xa_audit_db_flavor == 'sqla': + xa_audit_hdfs_is_enabled = config['configurations']['ranger-hbase-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else False + ssl_keystore_password = config['configurations']['ranger-hbase-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password'] if xml_configurations_supported else None + ssl_truststore_password = config['configurations']['ranger-hbase-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password'] if xml_configurations_supported else None + credential_file = format('/etc/ranger/{repo_name}/cred.jceks') + + # for SQLA explicitly disable audit to DB for Ranger + if has_ranger_admin and stack_supports_ranger_audit_db and xa_audit_db_flavor.lower() == 'sqla': xa_audit_db_is_enabled = False +# ranger hbase plugin section end create_hbase_home_directory = check_stack_feature(StackFeature.HBASE_HOME_DIRECTORY, stack_version_formatted) hbase_home_directory = format("/user/{hbase_user}") http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py index 0d73e39..d32dce1 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py @@ -22,7 +22,7 @@ from resource_management.core.logger import Logger def setup_ranger_hbase(upgrade_type=None, service_name="hbase-master"): import params - if params.has_ranger_admin: + if params.enable_ranger_hbase: stack_version = None @@ -103,4 +103,4 @@ def setup_ranger_hbase(upgrade_type=None, service_name="hbase-master"): ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password, stack_version_override = stack_version, skip_if_rangeradmin_down= not params.retryAble) else: - Logger.info('Ranger admin not installed') + Logger.info('Ranger HBase plugin is not enabled') http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py index 21e7b68..31431b9 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py @@ -44,7 +44,7 @@ from resource_management.libraries.functions.get_lzo_packages import get_lzo_pac from resource_management.libraries.functions.hdfs_utils import is_https_enabled_in_hdfs from resource_management.libraries.functions import is_empty from resource_management.libraries.functions.get_architecture import get_architecture - +from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs config = Script.get_config() tmp_dir = Script.get_tmp_dir() @@ -392,95 +392,100 @@ dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize'] mapred_pid_dir_prefix = default("/configurations/mapred-env/mapred_pid_dir_prefix","/var/run/hadoop-mapreduce") mapred_log_dir_prefix = default("/configurations/mapred-env/mapred_log_dir_prefix","/var/log/hadoop-mapreduce") -# ranger host -ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) -has_ranger_admin = not len(ranger_admin_hosts) == 0 -xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported'] -ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] - -#ranger hdfs properties -policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url'] -if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'): - policymgr_mgr_url = policymgr_mgr_url.rstrip('/') -xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits') -xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger') -xa_db_host = config['configurations']['admin-properties']['db_host'] -repo_name = str(config['clusterName']) + '_hadoop' -repo_name_value = config['configurations']['ranger-hdfs-security']['ranger.plugin.hdfs.service.name'] -if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}": - repo_name = repo_name_value - hadoop_security_authentication = config['configurations']['core-site']['hadoop.security.authentication'] hadoop_security_authorization = config['configurations']['core-site']['hadoop.security.authorization'] fs_default_name = config['configurations']['core-site']['fs.defaultFS'] hadoop_security_auth_to_local = config['configurations']['core-site']['hadoop.security.auth_to_local'] -hadoop_rpc_protection = config['configurations']['ranger-hdfs-plugin-properties']['hadoop.rpc.protection'] -common_name_for_certificate = config['configurations']['ranger-hdfs-plugin-properties']['common.name.for.certificate'] - -repo_config_username = config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_USERNAME'] if security_enabled: sn_principal_name = default("/configurations/hdfs-site/dfs.secondary.namenode.kerberos.principal", "nn/[email protected]") sn_principal_name = sn_principal_name.replace('_HOST',hostname.lower()) -ranger_env = config['configurations']['ranger-env'] -ranger_plugin_properties = config['configurations']['ranger-hdfs-plugin-properties'] -policy_user = config['configurations']['ranger-hdfs-plugin-properties']['policy_user'] - -#For curl command in ranger plugin to get db connector +# for curl command in ranger plugin to get db connector jdk_location = config['hostLevelParams']['jdk_location'] java_share_dir = '/usr/share/java' is_https_enabled = is_https_enabled_in_hdfs(config['configurations']['hdfs-site']['dfs.http.policy'], config['configurations']['hdfs-site']['dfs.https.enable']) -if has_ranger_admin: - enable_ranger_hdfs = (config['configurations']['ranger-hdfs-plugin-properties']['ranger-hdfs-plugin-enabled'].lower() == 'yes') +# ranger hdfs plugin section start + +# ranger host +ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) +has_ranger_admin = not len(ranger_admin_hosts) == 0 + +# ranger support xml_configuration flag, instead of depending on ranger xml_configurations_supported/ranger-env, using stack feature +xml_configurations_supported = check_stack_feature(StackFeature.RANGER_XML_CONFIGURATION, version_for_stack_feature_checks) + +# ambari-server hostname +ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] + +# ranger hdfs plugin enabled property +enable_ranger_hdfs = default("/configurations/ranger-hdfs-plugin-properties/ranger-hdfs-plugin-enabled", "No") +enable_ranger_hdfs = True if enable_ranger_hdfs.lower() == 'yes' else False + +# get ranger hdfs properties if enable_ranger_hdfs is True +if enable_ranger_hdfs: + # ranger policy url + policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url'] + if xml_configurations_supported: + policymgr_mgr_url = config['configurations']['ranger-hdfs-security']['ranger.plugin.hdfs.policy.rest.url'] + + if not is_empty(policymgr_mgr_url) and policymgr_mgr_url.endswith('/'): + policymgr_mgr_url = policymgr_mgr_url.rstrip('/') + + # ranger audit db user + xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger') + + # ranger hdfs service name + repo_name = str(config['clusterName']) + '_hadoop' + repo_name_value = config['configurations']['ranger-hdfs-security']['ranger.plugin.hdfs.service.name'] + if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}": + repo_name = repo_name_value + + hadoop_rpc_protection = config['configurations']['ranger-hdfs-plugin-properties']['hadoop.rpc.protection'] + common_name_for_certificate = config['configurations']['ranger-hdfs-plugin-properties']['common.name.for.certificate'] + repo_config_username = config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_USERNAME'] + + # ranger-env config + ranger_env = config['configurations']['ranger-env'] + + # create ranger-env config having external ranger credential properties + if not has_ranger_admin and enable_ranger_hdfs: + external_admin_username = default('/configurations/ranger-hdfs-plugin-properties/external_admin_username', 'admin') + external_admin_password = default('/configurations/ranger-hdfs-plugin-properties/external_admin_password', 'admin') + external_ranger_admin_username = default('/configurations/ranger-hdfs-plugin-properties/external_ranger_admin_username', 'amb_ranger_admin') + external_ranger_admin_password = default('/configurations/ranger-hdfs-plugin-properties/external_ranger_admin_password', 'amb_ranger_admin') + ranger_env = {} + ranger_env['admin_username'] = external_admin_username + ranger_env['admin_password'] = external_admin_password + ranger_env['ranger_admin_username'] = external_ranger_admin_username + ranger_env['ranger_admin_password'] = external_ranger_admin_password + + ranger_plugin_properties = config['configurations']['ranger-hdfs-plugin-properties'] + policy_user = config['configurations']['ranger-hdfs-plugin-properties']['policy_user'] + repo_config_password = config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_PASSWORD'] + xa_audit_db_password = '' - if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db: - xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) - repo_config_password = unicode(config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']) - xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower() + if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db and has_ranger_admin: + xa_audit_db_password = config['configurations']['admin-properties']['audit_db_password'] + + downloaded_custom_connector = None previous_jdbc_jar_name = None + driver_curl_source = None + driver_curl_target = None + previous_jdbc_jar = None + + # to get db connector related properties + if has_ranger_admin and stack_supports_ranger_audit_db: + xa_audit_db_flavor = config['configurations']['admin-properties']['DB_FLAVOR'] + jdbc_jar_name, previous_jdbc_jar_name, audit_jdbc_url, jdbc_driver = get_audit_configs(config) - if stack_supports_ranger_audit_db: - - if xa_audit_db_flavor == 'mysql': - jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None) - previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None) - audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}') - jdbc_driver = "com.mysql.jdbc.Driver" - elif xa_audit_db_flavor == 'oracle': - jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name", None) - previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None) - colon_count = xa_db_host.count(':') - if colon_count == 2 or colon_count == 0: - audit_jdbc_url = format('jdbc:oracle:thin:@{xa_db_host}') - else: - audit_jdbc_url = format('jdbc:oracle:thin:@//{xa_db_host}') - jdbc_driver = "oracle.jdbc.OracleDriver" - elif xa_audit_db_flavor == 'postgres': - jdbc_jar_name = default("/hostLevelParams/custom_postgres_jdbc_name", None) - previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None) - audit_jdbc_url = format('jdbc:postgresql://{xa_db_host}/{xa_audit_db_name}') - jdbc_driver = "org.postgresql.Driver" - elif xa_audit_db_flavor == 'mssql': - jdbc_jar_name = default("/hostLevelParams/custom_mssql_jdbc_name", None) - previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None) - audit_jdbc_url = format('jdbc:sqlserver://{xa_db_host};databaseName={xa_audit_db_name}') - jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver" - elif xa_audit_db_flavor == 'sqla': - jdbc_jar_name = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None) - previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None) - audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}') - jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver" - - downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None - driver_curl_source = format("{jdk_location}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None - driver_curl_target = format("{hadoop_lib_home}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None - previous_jdbc_jar = format("{hadoop_lib_home}/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None - - sql_connector_jar = '' + downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None + driver_curl_source = format("{jdk_location}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None + driver_curl_target = format("{hadoop_lib_home}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None + previous_jdbc_jar = format("{hadoop_lib_home}/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None + sql_connector_jar = '' hdfs_ranger_plugin_config = { 'username': repo_config_username, @@ -504,6 +509,7 @@ if has_ranger_admin: 'repositoryType': 'hdfs', 'assetType': '1' } + if stack_supports_ranger_kerberos and security_enabled: hdfs_ranger_plugin_config['policy.download.auth.users'] = hdfs_user hdfs_ranger_plugin_config['tag.download.auth.users'] = hdfs_user @@ -520,14 +526,16 @@ if has_ranger_admin: } xa_audit_db_is_enabled = False - ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls'] if xml_configurations_supported and stack_supports_ranger_audit_db: xa_audit_db_is_enabled = config['configurations']['ranger-hdfs-audit']['xasecure.audit.destination.db'] - xa_audit_hdfs_is_enabled = config['configurations']['ranger-hdfs-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None - ssl_keystore_password = unicode(config['configurations']['ranger-hdfs-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None - ssl_truststore_password = unicode(config['configurations']['ranger-hdfs-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None - credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None - #For SQLA explicitly disable audit to DB for Ranger - if xa_audit_db_flavor == 'sqla': + xa_audit_hdfs_is_enabled = config['configurations']['ranger-hdfs-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else False + ssl_keystore_password = config['configurations']['ranger-hdfs-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password'] if xml_configurations_supported else None + ssl_truststore_password = config['configurations']['ranger-hdfs-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password'] if xml_configurations_supported else None + credential_file = format('/etc/ranger/{repo_name}/cred.jceks') + + # for SQLA explicitly disable audit to DB for Ranger + if has_ranger_admin and stack_supports_ranger_audit_db and xa_audit_db_flavor.lower() == 'sqla': xa_audit_db_is_enabled = False + +# ranger hdfs plugin section end http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py index e3aff9d..47c6e35 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py @@ -29,8 +29,7 @@ from resource_management.libraries.functions.format import format def setup_ranger_hdfs(upgrade_type=None): import params - if params.has_ranger_admin: - + if params.enable_ranger_hdfs: stack_version = None @@ -93,29 +92,28 @@ def setup_ranger_hdfs(upgrade_type=None): target_file = source_file + ".bak" Execute(("mv", source_file, target_file), sudo=True, only_if=format("test -f {source_file}")) else: - Logger.info('Ranger admin not installed') + Logger.info('Ranger Hdfs plugin is not enabled') def create_ranger_audit_hdfs_directories(): import params - if params.has_ranger_admin: - if params.xml_configurations_supported and params.enable_ranger_hdfs and params.xa_audit_hdfs_is_enabled: - params.HdfsResource("/ranger/audit", - type="directory", - action="create_on_execute", - owner=params.hdfs_user, - group=params.hdfs_user, - mode=0755, - recursive_chmod=True, - ) - params.HdfsResource("/ranger/audit/hdfs", - type="directory", - action="create_on_execute", - owner=params.hdfs_user, - group=params.hdfs_user, - mode=0700, - recursive_chmod=True, - ) - params.HdfsResource(None, action="execute") + if params.enable_ranger_hdfs and params.xml_configurations_supported and params.xa_audit_hdfs_is_enabled: + params.HdfsResource("/ranger/audit", + type="directory", + action="create_on_execute", + owner=params.hdfs_user, + group=params.hdfs_user, + mode=0755, + recursive_chmod=True, + ) + params.HdfsResource("/ranger/audit/hdfs", + type="directory", + action="create_on_execute", + owner=params.hdfs_user, + group=params.hdfs_user, + mode=0700, + recursive_chmod=True, + ) + params.HdfsResource(None, action="execute") else: - Logger.info('Ranger admin not installed') + Logger.info('Ranger Hdfs plugin is not enabled') http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index 62fdbfd..9185f78 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -43,6 +43,7 @@ from resource_management.libraries.functions.expect import expect from resource_management.libraries import functions from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_from_hostnames +from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs # Default log4j version; put config files under /etc/hive/conf log4j_version = '1' @@ -641,84 +642,85 @@ if has_hive_interactive: hive_server2_zookeeper_namespace = config['configurations']['hive-site']['hive.server2.zookeeper.namespace'] hive_zookeeper_quorum = config['configurations']['hive-site']['hive.zookeeper.quorum'] -# ranger host -ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) -has_ranger_admin = not len(ranger_admin_hosts) == 0 -xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported'] - -#ranger hive properties -policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url'] -if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'): - policymgr_mgr_url = policymgr_mgr_url.rstrip('/') -xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits') -xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger') -xa_db_host = config['configurations']['admin-properties']['db_host'] -repo_name = str(config['clusterName']) + '_hive' -repo_name_value = config['configurations']['ranger-hive-security']['ranger.plugin.hive.service.name'] -if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}": - repo_name = repo_name_value - -jdbc_driver_class_name = config['configurations']['ranger-hive-plugin-properties']['jdbc.driverClassName'] -common_name_for_certificate = config['configurations']['ranger-hive-plugin-properties']['common.name.for.certificate'] - -repo_config_username = config['configurations']['ranger-hive-plugin-properties']['REPOSITORY_CONFIG_USERNAME'] - -ranger_env = config['configurations']['ranger-env'] -ranger_plugin_properties = config['configurations']['ranger-hive-plugin-properties'] -policy_user = config['configurations']['ranger-hive-plugin-properties']['policy_user'] +if security_enabled: + hive_principal = hive_server_principal.replace('_HOST',hostname.lower()) + hive_keytab = config['configurations']['hive-site']['hive.server2.authentication.kerberos.keytab'] hive_cluster_token_zkstore = default("/configurations/hive-site/hive.cluster.delegation.token.store.zookeeper.znode", None) jaas_file = os.path.join(hive_config_dir, 'zkmigrator_jaas.conf') zkdtsm_pattern = '/zkdtsm_*' hive_zk_namespace = default("/configurations/hive-site/hive.zookeeper.namespace", None) -if security_enabled: - hive_principal = hive_server_principal.replace('_HOST',hostname.lower()) - hive_keytab = config['configurations']['hive-site']['hive.server2.authentication.kerberos.keytab'] +# ranger hive plugin section start + +# ranger host +ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) +has_ranger_admin = not len(ranger_admin_hosts) == 0 -#For curl command in ranger plugin to get db connector -if has_ranger_admin: - enable_ranger_hive = (config['configurations']['hive-env']['hive_security_authorization'].lower() == 'ranger') - repo_config_password = unicode(config['configurations']['ranger-hive-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']) - xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower() +# ranger hive plugin enabled property +enable_ranger_hive = config['configurations']['hive-env']['hive_security_authorization'].lower() == 'ranger' + +# ranger support xml_configuration flag, instead of depending on ranger xml_configurations_supported/ranger-env, using stack feature +xml_configurations_supported = check_stack_feature(StackFeature.RANGER_XML_CONFIGURATION, version_for_stack_feature_checks) + +# get ranger hive properties if enable_ranger_hive is True +if enable_ranger_hive: + # get ranger policy url + policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url'] + if xml_configurations_supported: + policymgr_mgr_url = config['configurations']['ranger-hive-security']['ranger.plugin.hive.policy.rest.url'] + + if not is_empty(policymgr_mgr_url) and policymgr_mgr_url.endswith('/'): + policymgr_mgr_url = policymgr_mgr_url.rstrip('/') + + # ranger audit db user + xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger') + + # ranger hive service name + repo_name = str(config['clusterName']) + '_hive' + repo_name_value = config['configurations']['ranger-hive-security']['ranger.plugin.hive.service.name'] + if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}": + repo_name = repo_name_value + + jdbc_driver_class_name = config['configurations']['ranger-hive-plugin-properties']['jdbc.driverClassName'] + common_name_for_certificate = config['configurations']['ranger-hive-plugin-properties']['common.name.for.certificate'] + repo_config_username = config['configurations']['ranger-hive-plugin-properties']['REPOSITORY_CONFIG_USERNAME'] + + # ranger-env config + ranger_env = config['configurations']['ranger-env'] + + # create ranger-env config having external ranger credential properties + if not has_ranger_admin and enable_ranger_hive: + external_admin_username = default('/configurations/ranger-hive-plugin-properties/external_admin_username', 'admin') + external_admin_password = default('/configurations/ranger-hive-plugin-properties/external_admin_password', 'admin') + external_ranger_admin_username = default('/configurations/ranger-hive-plugin-properties/external_ranger_admin_username', 'amb_ranger_admin') + external_ranger_admin_password = default('/configurations/ranger-hive-plugin-properties/external_ranger_admin_password', 'amb_ranger_admin') + ranger_env = {} + ranger_env['admin_username'] = external_admin_username + ranger_env['admin_password'] = external_admin_password + ranger_env['ranger_admin_username'] = external_ranger_admin_username + ranger_env['ranger_admin_password'] = external_ranger_admin_password + + ranger_plugin_properties = config['configurations']['ranger-hive-plugin-properties'] + policy_user = config['configurations']['ranger-hive-plugin-properties']['policy_user'] + repo_config_password = config['configurations']['ranger-hive-plugin-properties']['REPOSITORY_CONFIG_PASSWORD'] + + ranger_downloaded_custom_connector = None ranger_previous_jdbc_jar_name = None + ranger_driver_curl_source = None + ranger_driver_curl_target = None + ranger_previous_jdbc_jar = None + + # to get db connector related properties + if has_ranger_admin and stack_supports_ranger_audit_db: + xa_audit_db_flavor = config['configurations']['admin-properties']['DB_FLAVOR'] + ranger_jdbc_jar_name, ranger_previous_jdbc_jar_name, audit_jdbc_url, jdbc_driver = get_audit_configs(config) - if stack_supports_ranger_audit_db: - if xa_audit_db_flavor and xa_audit_db_flavor == 'mysql': - ranger_jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None) - ranger_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None) - audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}') - jdbc_driver = "com.mysql.jdbc.Driver" - elif xa_audit_db_flavor and xa_audit_db_flavor == 'oracle': - ranger_jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name", None) - ranger_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None) - colon_count = xa_db_host.count(':') - if colon_count == 2 or colon_count == 0: - audit_jdbc_url = format('jdbc:oracle:thin:@{xa_db_host}') - else: - audit_jdbc_url = format('jdbc:oracle:thin:@//{xa_db_host}') - jdbc_driver = "oracle.jdbc.OracleDriver" - elif xa_audit_db_flavor and xa_audit_db_flavor == 'postgres': - ranger_jdbc_jar_name = default("/hostLevelParams/custom_postgres_jdbc_name", None) - ranger_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None) - audit_jdbc_url = format('jdbc:postgresql://{xa_db_host}/{xa_audit_db_name}') - jdbc_driver = "org.postgresql.Driver" - elif xa_audit_db_flavor and xa_audit_db_flavor == 'mssql': - ranger_jdbc_jar_name = default("/hostLevelParams/custom_mssql_jdbc_name", None) - ranger_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None) - audit_jdbc_url = format('jdbc:sqlserver://{xa_db_host};databaseName={xa_audit_db_name}') - jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver" - elif xa_audit_db_flavor and xa_audit_db_flavor == 'sqla': - ranger_jdbc_jar_name = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None) - ranger_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None) - audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}') - jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver" - - ranger_downloaded_custom_connector = format("{tmp_dir}/{ranger_jdbc_jar_name}") if stack_supports_ranger_audit_db else None - ranger_driver_curl_source = format("{jdk_location}/{ranger_jdbc_jar_name}") if stack_supports_ranger_audit_db else None - ranger_driver_curl_target = format("{hive_lib}/{ranger_jdbc_jar_name}") if stack_supports_ranger_audit_db else None - ranger_previous_jdbc_jar = format("{hive_lib}/{ranger_previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None - sql_connector_jar = '' + ranger_downloaded_custom_connector = format("{tmp_dir}/{ranger_jdbc_jar_name}") + ranger_driver_curl_source = format("{jdk_location}/{ranger_jdbc_jar_name}") + ranger_driver_curl_target = format("{hive_lib}/{ranger_jdbc_jar_name}") + ranger_previous_jdbc_jar = format("{hive_lib}/{ranger_previous_jdbc_jar_name}") + sql_connector_jar = '' ranger_hive_url = format("{hive_url}/default;principal={hive_principal}") if security_enabled else hive_url if stack_supports_ranger_hive_jdbc_url_change: @@ -757,20 +759,21 @@ if has_ranger_admin: 'type': 'hive' } + xa_audit_db_password = '' + if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db and has_ranger_admin: + xa_audit_db_password = config['configurations']['admin-properties']['audit_db_password'] xa_audit_db_is_enabled = False - xa_audit_db_password = '' - if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db: - xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) - ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls'] if xml_configurations_supported and stack_supports_ranger_audit_db: xa_audit_db_is_enabled = config['configurations']['ranger-hive-audit']['xasecure.audit.destination.db'] - xa_audit_hdfs_is_enabled = config['configurations']['ranger-hive-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None - ssl_keystore_password = unicode(config['configurations']['ranger-hive-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None - ssl_truststore_password = unicode(config['configurations']['ranger-hive-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None - credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None - #For SQLA explicitly disable audit to DB for Ranger - if xa_audit_db_flavor == 'sqla': + xa_audit_hdfs_is_enabled = config['configurations']['ranger-hive-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else False + ssl_keystore_password = config['configurations']['ranger-hive-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password'] if xml_configurations_supported else None + ssl_truststore_password = config['configurations']['ranger-hive-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password'] if xml_configurations_supported else None + credential_file = format('/etc/ranger/{repo_name}/cred.jceks') + + # for SQLA explicitly disable audit to DB for Ranger + if has_ranger_admin and stack_supports_ranger_audit_db and xa_audit_db_flavor.lower() == 'sqla': xa_audit_db_is_enabled = False +# ranger hive plugin section end \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py index 81a4e3e..80bd7c8 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py @@ -22,7 +22,7 @@ from resource_management.core.logger import Logger def setup_ranger_hive(upgrade_type = None): import params - if params.has_ranger_admin: + if params.enable_ranger_hive: stack_version = None @@ -34,7 +34,7 @@ def setup_ranger_hive(upgrade_type = None): else: Logger.info("Hive: Setup ranger: command retry not enabled thus skipping if ranger admin is down !") - if params.xml_configurations_supported and params.enable_ranger_hive and params.xa_audit_hdfs_is_enabled: + if params.xml_configurations_supported and params.xa_audit_hdfs_is_enabled: params.HdfsResource("/ranger/audit", type="directory", action="create_on_execute", @@ -95,4 +95,4 @@ def setup_ranger_hive(upgrade_type = None): ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password, stack_version_override = stack_version, skip_if_rangeradmin_down= not params.retryAble) else: - Logger.info('Ranger admin not installed') + Logger.info('Ranger Hive plugin is not enabled') http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py index 82849c8..6c7ff69 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py @@ -34,6 +34,7 @@ from resource_management.libraries.functions import stack_select from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources +from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs # server configurations config = Script.get_config() @@ -166,41 +167,66 @@ else: kafka_jaas_principal = None kafka_keytab_path = None -# *********************** RANGER PLUGIN CHANGES *********************** +# for curl command in ranger plugin to get db connector +jdk_location = config['hostLevelParams']['jdk_location'] + +# ranger kafka plugin section start + # ranger host -# ********************************************************************** ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) has_ranger_admin = not len(ranger_admin_hosts) == 0 -xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported'] + +# ranger support xml_configuration flag, instead of depending on ranger xml_configurations_supported/ranger-env, using stack feature +xml_configurations_supported = check_stack_feature(StackFeature.RANGER_XML_CONFIGURATION, version_for_stack_feature_checks) + +# ambari-server hostname ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] ranger_admin_log_dir = default("/configurations/ranger-env/ranger_admin_log_dir","/var/log/ranger/admin") -is_supported_kafka_ranger = config['configurations']['kafka-env']['is_supported_kafka_ranger'] -#ranger kafka properties -if has_ranger_admin and is_supported_kafka_ranger: +# ranger kafka plugin enabled property +enable_ranger_kafka = default("configurations/ranger-kafka-plugin-properties/ranger-kafka-plugin-enabled", "No") +enable_ranger_kafka = True if enable_ranger_kafka.lower() == 'yes' else False - enable_ranger_kafka = config['configurations']['ranger-kafka-plugin-properties']['ranger-kafka-plugin-enabled'] - enable_ranger_kafka = not is_empty(enable_ranger_kafka) and enable_ranger_kafka.lower() == 'yes' - policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url'] - if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'): +# ranger kafka-plugin supported flag, instead of dependending on is_supported_kafka_ranger/kafka-env.xml, using stack feature +is_supported_kafka_ranger = check_stack_feature(StackFeature.KAFKA_RANGER_PLUGIN_SUPPORT, version_for_stack_feature_checks) + +# ranger kafka properties +if enable_ranger_kafka and is_supported_kafka_ranger: + # get ranger policy url + policymgr_mgr_url = config['configurations']['ranger-kafka-security']['ranger.plugin.kafka.policy.rest.url'] + + if not is_empty(policymgr_mgr_url) and policymgr_mgr_url.endswith('/'): policymgr_mgr_url = policymgr_mgr_url.rstrip('/') - xa_audit_db_flavor = config['configurations']['admin-properties']['DB_FLAVOR'] - xa_audit_db_flavor = xa_audit_db_flavor.lower() if xa_audit_db_flavor else None - xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits') + + # ranger audit db user xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger') + xa_audit_db_password = '' - if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db: - xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) - xa_db_host = config['configurations']['admin-properties']['db_host'] + if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db and has_ranger_admin: + xa_audit_db_password = config['configurations']['admin-properties']['audit_db_password'] + + # ranger kafka service/repository name repo_name = str(config['clusterName']) + '_kafka' repo_name_value = config['configurations']['ranger-kafka-security']['ranger.plugin.kafka.service.name'] if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}": repo_name = repo_name_value ranger_env = config['configurations']['ranger-env'] - ranger_plugin_properties = config['configurations']['ranger-kafka-plugin-properties'] + # create ranger-env config having external ranger credential properties + if not has_ranger_admin and enable_ranger_kafka: + external_admin_username = default('/configurations/ranger-kafka-plugin-properties/external_admin_username', 'admin') + external_admin_password = default('/configurations/ranger-kafka-plugin-properties/external_admin_password', 'admin') + external_ranger_admin_username = default('/configurations/ranger-kafka-plugin-properties/external_ranger_admin_username', 'amb_ranger_admin') + external_ranger_admin_password = default('/configurations/ranger-kafka-plugin-properties/external_ranger_admin_password', 'amb_ranger_admin') + ranger_env = {} + ranger_env['admin_username'] = external_admin_username + ranger_env['admin_password'] = external_admin_password + ranger_env['ranger_admin_username'] = external_ranger_admin_username + ranger_env['ranger_admin_password'] = external_ranger_admin_password + + ranger_plugin_properties = config['configurations']['ranger-kafka-plugin-properties'] ranger_kafka_audit = config['configurations']['ranger-kafka-audit'] ranger_kafka_audit_attrs = config['configuration_attributes']['ranger-kafka-audit'] ranger_kafka_security = config['configurations']['ranger-kafka-security'] @@ -212,7 +238,7 @@ if has_ranger_admin and is_supported_kafka_ranger: ranger_plugin_config = { 'username' : config['configurations']['ranger-kafka-plugin-properties']['REPOSITORY_CONFIG_USERNAME'], - 'password' : unicode(config['configurations']['ranger-kafka-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']), + 'password' : config['configurations']['ranger-kafka-plugin-properties']['REPOSITORY_CONFIG_PASSWORD'], 'zookeeper.connect' : config['configurations']['ranger-kafka-plugin-properties']['zookeeper.connect'], 'commonNameForCertificate' : config['configurations']['ranger-kafka-plugin-properties']['common.name.for.certificate'] } @@ -232,64 +258,40 @@ if has_ranger_admin and is_supported_kafka_ranger: ranger_plugin_config['tag.download.auth.users'] = kafka_user ranger_plugin_config['ambari.service.check.user'] = policy_user - #For curl command in ranger plugin to get db connector - jdk_location = config['hostLevelParams']['jdk_location'] - java_share_dir = '/usr/share/java' + downloaded_custom_connector = None previous_jdbc_jar_name = None + driver_curl_source = None + driver_curl_target = None + previous_jdbc_jar = None - if stack_supports_ranger_audit_db: - if xa_audit_db_flavor and xa_audit_db_flavor == 'mysql': - jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None) - previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None) - audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}') - jdbc_driver = "com.mysql.jdbc.Driver" - elif xa_audit_db_flavor and xa_audit_db_flavor == 'oracle': - jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name", None) - previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None) - colon_count = xa_db_host.count(':') - if colon_count == 2 or colon_count == 0: - audit_jdbc_url = format('jdbc:oracle:thin:@{xa_db_host}') - else: - audit_jdbc_url = format('jdbc:oracle:thin:@//{xa_db_host}') - jdbc_driver = "oracle.jdbc.OracleDriver" - elif xa_audit_db_flavor and xa_audit_db_flavor == 'postgres': - jdbc_jar_name = default("/hostLevelParams/custom_postgres_jdbc_name", None) - previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None) - audit_jdbc_url = format('jdbc:postgresql://{xa_db_host}/{xa_audit_db_name}') - jdbc_driver = "org.postgresql.Driver" - elif xa_audit_db_flavor and xa_audit_db_flavor == 'mssql': - jdbc_jar_name = default("/hostLevelParams/custom_mssql_jdbc_name", None) - previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None) - audit_jdbc_url = format('jdbc:sqlserver://{xa_db_host};databaseName={xa_audit_db_name}') - jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver" - elif xa_audit_db_flavor and xa_audit_db_flavor == 'sqla': - jdbc_jar_name = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None) - previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None) - audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}') - jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver" - - downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None - driver_curl_source = format("{jdk_location}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None - driver_curl_target = format("{kafka_home}/libs/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None - previous_jdbc_jar = format("{kafka_home}/libs/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None + if has_ranger_admin and stack_supports_ranger_audit_db: + xa_audit_db_flavor = config['configurations']['admin-properties']['DB_FLAVOR'] + jdbc_jar_name, previous_jdbc_jar_name, audit_jdbc_url, jdbc_driver = get_audit_configs(config) + + downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None + driver_curl_source = format("{jdk_location}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None + driver_curl_target = format("{kafka_home}/libs/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None + previous_jdbc_jar = format("{kafka_home}/libs/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None xa_audit_db_is_enabled = False - ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls'] if xml_configurations_supported and stack_supports_ranger_audit_db: xa_audit_db_is_enabled = config['configurations']['ranger-kafka-audit']['xasecure.audit.destination.db'] + xa_audit_hdfs_is_enabled = default('/configurations/ranger-kafka-audit/xasecure.audit.destination.hdfs', False) - ssl_keystore_password = unicode(config['configurations']['ranger-kafka-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None - ssl_truststore_password = unicode(config['configurations']['ranger-kafka-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None - credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None + ssl_keystore_password = config['configurations']['ranger-kafka-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password'] if xml_configurations_supported else None + ssl_truststore_password = config['configurations']['ranger-kafka-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password'] if xml_configurations_supported else None + credential_file = format('/etc/ranger/{repo_name}/cred.jceks') stack_version = get_stack_version('kafka-broker') setup_ranger_env_sh_source = format('{stack_root}/{stack_version}/ranger-kafka-plugin/install/conf.templates/enable/kafka-ranger-env.sh') setup_ranger_env_sh_target = format("{conf_dir}/kafka-ranger-env.sh") - #For SQLA explicitly disable audit to DB for Ranger - if xa_audit_db_flavor == 'sqla': + # for SQLA explicitly disable audit to DB for Ranger + if has_ranger_admin and stack_supports_ranger_audit_db and xa_audit_db_flavor.lower() == 'sqla': xa_audit_db_is_enabled = False +# ranger kafka plugin section end + namenode_hosts = default("/clusterHostInfo/namenode_host", []) has_namenode = not len(namenode_hosts) == 0 http://git-wip-us.apache.org/repos/asf/ambari/blob/1524fd77/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py index 528dec2..e9719aa 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py @@ -22,7 +22,7 @@ from resource_management.libraries.functions.setup_ranger_plugin_xml import setu def setup_ranger_kafka(): import params - if params.has_ranger_admin: + if params.enable_ranger_kafka: from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin @@ -87,4 +87,4 @@ def setup_ranger_kafka(): else: Logger.info("Stack does not support core-site.xml creation for Ranger plugin, skipping core-site.xml configurations") else: - Logger.info('Ranger admin not installed') + Logger.info('Ranger Kafka plugin is not enabled')
