AMBARI-21010. HDP 3.0 TP - create Service Advisor for Ranger_KMS.(vbrodetsky)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e1265262 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e1265262 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e1265262 Branch: refs/heads/branch-feature-AMBARI-20859 Commit: e12652628102bf08b23e58b1954929b102c688df Parents: ad09bb6 Author: Vitaly Brodetskyi <[email protected]> Authored: Mon May 15 14:51:05 2017 +0300 Committer: Vitaly Brodetskyi <[email protected]> Committed: Mon May 15 14:51:05 2017 +0300 ---------------------------------------------------------------------- .../RANGER_KMS/0.5.0.3.0/service_advisor.py | 281 +++++++++++++++++++ 1 file changed, 281 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/e1265262/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/service_advisor.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/service_advisor.py new file mode 100644 index 0000000..9c33218 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/service_advisor.py @@ -0,0 +1,281 @@ +#!/usr/bin/env ambari-python-wrap +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +# Python imports +import imp +import os +import traceback +import re +import socket +import fnmatch + + +from resource_management.core.logger import Logger + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/') +PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py') + +try: + with open(PARENT_FILE, 'rb') as fp: + service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE)) +except Exception as e: + traceback.print_exc() + print "Failed to load parent" + +class RangerKMSServiceAdvisor(service_advisor.ServiceAdvisor): + + def __init__(self, *args, **kwargs): + self.as_super = super(RangerKMSServiceAdvisor, self) + self.as_super.__init__(*args, **kwargs) + + # Always call these methods + self.modifyMastersWithMultipleInstances() + self.modifyCardinalitiesDict() + self.modifyHeapSizeProperties() + self.modifyNotValuableComponents() + self.modifyComponentsNotPreferableOnServer() + self.modifyComponentLayoutSchemes() + + def modifyMastersWithMultipleInstances(self): + """ + Modify the set of masters with multiple instances. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyCardinalitiesDict(self): + """ + Modify the dictionary of cardinalities. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyHeapSizeProperties(self): + """ + Modify the dictionary of heap size properties. + Must be overriden in child class. + """ + pass + + def modifyNotValuableComponents(self): + """ + Modify the set of components whose host assignment is based on other services. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyComponentsNotPreferableOnServer(self): + """ + Modify the set of components that are not preferable on the server. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyComponentLayoutSchemes(self): + """ + Modify layout scheme dictionaries for components. + The scheme dictionary basically maps the number of hosts to + host index where component should exist. + Must be overriden in child class. + """ + # Nothing to do + pass + + def getServiceComponentLayoutValidations(self, services, hosts): + """ + Get a list of errors. + Must be overriden in child class. + """ + + return [] + + def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts): + """ + Entry point. + Must be overriden in child class. + """ + #Logger.info("Class: %s, Method: %s. Recommending Service Configurations." % + # (self.__class__.__name__, inspect.stack()[0][3])) + + recommender = RangerKMSRecommender() + recommender.recommendRangerKMSConfigurationsFromHDP23(configurations, clusterData, services, hosts) + recommender.recommendRangerKMSConfigurationsFromHDP25(configurations, clusterData, services, hosts) + + + + def getServiceConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts): + """ + Entry point. + Validate configurations for the service. Return a list of errors. + The code for this function should be the same for each Service Advisor. + """ + #Logger.info("Class: %s, Method: %s. Validating Configurations." % + # (self.__class__.__name__, inspect.stack()[0][3])) + + validator = RangerKMSValidator() + # Calls the methods of the validator using arguments, + # method(siteProperties, siteRecommendations, configurations, services, hosts) + return validator.validateListOfConfigUsingMethod(configurations, recommendedDefaults, services, hosts, validator.validators) + + + +class RangerKMSRecommender(service_advisor.ServiceAdvisor): + """ + RangerKMS Recommender suggests properties when adding the service for the first time or modifying configs via the UI. + """ + + def __init__(self, *args, **kwargs): + self.as_super = super(RangerKMSRecommender, self) + self.as_super.__init__(*args, **kwargs) + + + def recommendRangerKMSConfigurationsFromHDP23(self, configurations, clusterData, services, hosts): + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + putRangerKmsDbksProperty = self.putProperty(configurations, "dbks-site", services) + putRangerKmsProperty = self.putProperty(configurations, "kms-properties", services) + kmsEnvProperties = self.getSiteProperties(services['configurations'], 'kms-env') + putCoreSiteProperty = self.putProperty(configurations, "core-site", services) + putCoreSitePropertyAttribute = self.putPropertyAttribute(configurations, "core-site") + putRangerKmsAuditProperty = self.putProperty(configurations, "ranger-kms-audit", services) + security_enabled = self.isSecurityEnabled(services) + putRangerKmsSiteProperty = self.putProperty(configurations, "kms-site", services) + putRangerKmsSitePropertyAttribute = self.putPropertyAttribute(configurations, "kms-site") + + if 'kms-properties' in services['configurations'] and ('DB_FLAVOR' in services['configurations']['kms-properties']['properties']): + + rangerKmsDbFlavor = services['configurations']["kms-properties"]["properties"]["DB_FLAVOR"] + + if ('db_host' in services['configurations']['kms-properties']['properties']) and ('db_name' in services['configurations']['kms-properties']['properties']): + + rangerKmsDbHost = services['configurations']["kms-properties"]["properties"]["db_host"] + rangerKmsDbName = services['configurations']["kms-properties"]["properties"]["db_name"] + + ranger_kms_db_url_dict = { + 'MYSQL': {'ranger.ks.jpa.jdbc.driver': 'com.mysql.jdbc.Driver', + 'ranger.ks.jpa.jdbc.url': 'jdbc:mysql://' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + '/' + rangerKmsDbName}, + 'ORACLE': {'ranger.ks.jpa.jdbc.driver': 'oracle.jdbc.driver.OracleDriver', + 'ranger.ks.jpa.jdbc.url': 'jdbc:oracle:thin:@' + self.getOracleDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost, rangerKmsDbName)}, + 'POSTGRES': {'ranger.ks.jpa.jdbc.driver': 'org.postgresql.Driver', + 'ranger.ks.jpa.jdbc.url': 'jdbc:postgresql://' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + '/' + rangerKmsDbName}, + 'MSSQL': {'ranger.ks.jpa.jdbc.driver': 'com.microsoft.sqlserver.jdbc.SQLServerDriver', + 'ranger.ks.jpa.jdbc.url': 'jdbc:sqlserver://' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + ';databaseName=' + rangerKmsDbName}, + 'SQLA': {'ranger.ks.jpa.jdbc.driver': 'sap.jdbc4.sqlanywhere.IDriver', + 'ranger.ks.jpa.jdbc.url': 'jdbc:sqlanywhere:host=' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + ';database=' + rangerKmsDbName} + } + + rangerKmsDbProperties = ranger_kms_db_url_dict.get(rangerKmsDbFlavor, ranger_kms_db_url_dict['MYSQL']) + for key in rangerKmsDbProperties: + putRangerKmsDbksProperty(key, rangerKmsDbProperties.get(key)) + + if kmsEnvProperties and self.checkSiteProperties(kmsEnvProperties, 'kms_user') and 'KERBEROS' in servicesList: + kmsUser = kmsEnvProperties['kms_user'] + kmsUserOld = self.getOldValue(services, 'kms-env', 'kms_user') + self.put_proxyuser_value(kmsUser, '*', is_groups=True, services=services, configurations=configurations, put_function=putCoreSiteProperty) + if kmsUserOld is not None and kmsUser != kmsUserOld: + putCoreSitePropertyAttribute("hadoop.proxyuser.{0}.groups".format(kmsUserOld), 'delete', 'true') + services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.groups".format(kmsUserOld)}) + services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.groups".format(kmsUser)}) + + if "HDFS" in servicesList: + if 'core-site' in services['configurations'] and ('fs.defaultFS' in services['configurations']['core-site']['properties']): + default_fs = services['configurations']['core-site']['properties']['fs.defaultFS'] + putRangerKmsAuditProperty('xasecure.audit.destination.hdfs.dir', '{0}/{1}/{2}'.format(default_fs,'ranger','audit')) + + required_services = [{'service' : 'YARN', 'config-type': 'yarn-env', 'property-name': 'yarn_user', 'proxy-category': ['hosts', 'users', 'groups']}, + {'service' : 'SPARK', 'config-type': 'livy-env', 'property-name': 'livy_user', 'proxy-category': ['hosts', 'users', 'groups']}] + + required_services_for_secure = [{'service' : 'HIVE', 'config-type': 'hive-env', 'property-name': 'hive_user', 'proxy-category': ['hosts', 'users']}, + {'service' : 'OOZIE', 'config-type': 'oozie-env', 'property-name': 'oozie_user', 'proxy-category': ['hosts', 'users']}] + + if security_enabled: + required_services.extend(required_services_for_secure) + + # recommendations for kms proxy related properties + self.recommendKMSProxyUsers(configurations, services, hosts, required_services) + + ambari_user = self.getAmbariUser(services) + if security_enabled: + # adding for ambari user + putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.users'.format(ambari_user), '*') + putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.hosts'.format(ambari_user), '*') + # adding for HTTP + putRangerKmsSiteProperty('hadoop.kms.proxyuser.HTTP.users', '*') + putRangerKmsSiteProperty('hadoop.kms.proxyuser.HTTP.hosts', '*') + else: + self.deleteKMSProxyUsers(configurations, services, hosts, required_services_for_secure) + # deleting ambari user proxy properties + putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.hosts'.format(ambari_user), 'delete', 'true') + putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.users'.format(ambari_user), 'delete', 'true') + # deleting HTTP proxy properties + putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.HTTP.hosts', 'delete', 'true') + putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.HTTP.users', 'delete', 'true') + + + def recommendRangerKMSConfigurationsFromHDP25(self, configurations, clusterData, services, hosts): + + security_enabled = self.isSecurityEnabled(services) + required_services = [{'service' : 'RANGER', 'config-type': 'ranger-env', 'property-name': 'ranger_user', 'proxy-category': ['hosts', 'users', 'groups']}] + + if security_enabled: + # recommendations for kms proxy related properties + self.recommendKMSProxyUsers(configurations, services, hosts, required_services) + else: + self.deleteKMSProxyUsers(configurations, services, hosts, required_services) + + + + def recommendRangerKMSConfigurations(self, configurations, clusterData, services, hosts): + putRangerKmsEnvProperty = self.putProperty(configurations, "kms-env", services) + + ranger_kms_ssl_enabled = False + ranger_kms_ssl_port = "9393" + if 'ranger-kms-site' in services['configurations'] and 'ranger.service.https.attrib.ssl.enabled' in services['configurations']['ranger-kms-site']['properties']: + ranger_kms_ssl_enabled = services['configurations']['ranger-kms-site']['properties']['ranger.service.https.attrib.ssl.enabled'].lower() == "true" + + if 'ranger-kms-site' in services['configurations'] and 'ranger.service.https.port' in services['configurations']['ranger-kms-site']['properties']: + ranger_kms_ssl_port = services['configurations']['ranger-kms-site']['properties']['ranger.service.https.port'] + + if ranger_kms_ssl_enabled: + putRangerKmsEnvProperty("kms_port", ranger_kms_ssl_port) + else: + putRangerKmsEnvProperty("kms_port", "9292") + + + +class RangerKMSValidator(service_advisor.ServiceAdvisor): + """ + RangerKMS Validator checks the correctness of properties whenever the service is first added or the user attempts to + change configs via the UI. + """ + + def __init__(self, *args, **kwargs): + self.as_super = super(RangerKMSValidator, self) + self.as_super.__init__(*args, **kwargs) + + self.validators = [] + + + + +
