http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_23.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_23.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_23.py new file mode 100755 index 0000000..d8b6729 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_23.py @@ -0,0 +1,995 @@ +#!/usr/bin/env ambari-python-wrap +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +# Python Imports +import os +import re +import fnmatch +import math +import socket + +# Local Imports +from resource_management.core.logger import Logger + +try: + from stack_advisor_22 import * +except ImportError: + #Ignore ImportError + print("stack_advisor_22 not found") + +DB_TYPE_DEFAULT_PORT_MAP = {"MYSQL":"3306", "ORACLE":"1521", "POSTGRES":"5432", "MSSQL":"1433", "SQLA":"2638"} + +class HDP23StackAdvisor(HDP22StackAdvisor): + + def __init__(self): + super(HDP23StackAdvisor, self).__init__() + Logger.initialize_logger() + + def getComponentLayoutValidations(self, services, hosts): + parentItems = super(HDP23StackAdvisor, self).getComponentLayoutValidations(services, hosts) + + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist] + childItems = [] + + if "SPARK" in servicesList: + if "SPARK_THRIFTSERVER" in servicesList: + if not "HIVE_SERVER" in servicesList: + message = "SPARK_THRIFTSERVER requires HIVE services to be selected." + childItems.append( {"type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'SPARK_THRIFTSERVER'} ) + + hmsHosts = self.__getHosts(componentsList, "HIVE_METASTORE") if "HIVE" in servicesList else [] + sparkTsHosts = self.__getHosts(componentsList, "SPARK_THRIFTSERVER") if "SPARK" in servicesList else [] + + # if Spark Thrift Server is deployed but no Hive Server is deployed + if len(sparkTsHosts) > 0 and len(hmsHosts) == 0: + message = "SPARK_THRIFTSERVER requires HIVE_METASTORE to be selected/deployed." + childItems.append( { "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'SPARK_THRIFTSERVER' } ) + + parentItems.extend(childItems) + return parentItems + + def __getHosts(self, componentsList, componentName): + host_lists = [component["hostnames"] for component in componentsList if + component["component_name"] == componentName] + if host_lists and len(host_lists) > 0: + return host_lists[0] + else: + return [] + + def getServiceConfigurationRecommenderDict(self): + parentRecommendConfDict = super(HDP23StackAdvisor, self).getServiceConfigurationRecommenderDict() + childRecommendConfDict = { + "TEZ": self.recommendTezConfigurations, + "HDFS": self.recommendHDFSConfigurations, + "YARN": self.recommendYARNConfigurations, + "HIVE": self.recommendHIVEConfigurations, + "HBASE": self.recommendHBASEConfigurations, + "KAFKA": self.recommendKAFKAConfigurations, + "RANGER": self.recommendRangerConfigurations, + "RANGER_KMS": self.recommendRangerKMSConfigurations, + "STORM": self.recommendStormConfigurations, + "SQOOP": self.recommendSqoopConfigurations + } + parentRecommendConfDict.update(childRecommendConfDict) + return parentRecommendConfDict + + def recommendTezConfigurations(self, configurations, clusterData, services, hosts): + super(HDP23StackAdvisor, self).recommendTezConfigurations(configurations, clusterData, services, hosts) + + putTezProperty = self.putProperty(configurations, "tez-site") + + if "HIVE" in self.getServiceNames(services): + if not "hive-site" in configurations: + self.recommendHIVEConfigurations(configurations, clusterData, services, hosts) + + if "hive-site" in configurations and "hive.tez.container.size" in configurations["hive-site"]["properties"]: + putTezProperty("tez.task.resource.memory.mb", configurations["hive-site"]["properties"]["hive.tez.container.size"]) + + # remove 2gb limit for tez.runtime.io.sort.mb + # in HDP 2.3 "tez.runtime.sorter.class" is set by default to PIPELINED, in other case comment calculation code below + taskResourceMemory = int(configurations["tez-site"]["properties"]["tez.task.resource.memory.mb"]) + # fit io.sort.mb into tenured regions + putTezProperty("tez.runtime.io.sort.mb", int(taskResourceMemory * 0.8 * 0.33)) + + if "tez-site" in services["configurations"] and "tez.runtime.sorter.class" in services["configurations"]["tez-site"]["properties"]: + if services["configurations"]["tez-site"]["properties"]["tez.runtime.sorter.class"] == "LEGACY": + putTezAttribute = self.putPropertyAttribute(configurations, "tez-site") + putTezAttribute("tez.runtime.io.sort.mb", "maximum", 1800) + pass + + serverProperties = services["ambari-server-properties"] + latest_tez_jar_version = None + + server_host = socket.getfqdn() + for host in hosts["items"]: + if server_host == host["Hosts"]["host_name"]: + server_host = host["Hosts"]["public_host_name"] + server_port = '8080' + server_protocol = 'http' + views_dir = '/var/lib/ambari-server/resources/views/' + + if serverProperties: + if 'client.api.port' in serverProperties: + server_port = serverProperties['client.api.port'] + if 'views.dir' in serverProperties: + views_dir = serverProperties['views.dir'] + if 'api.ssl' in serverProperties: + if serverProperties['api.ssl'].lower() == 'true': + server_protocol = 'https' + + views_work_dir = os.path.join(views_dir, 'work') + + if os.path.exists(views_work_dir) and os.path.isdir(views_work_dir): + last_version = '0.0.0' + for file in os.listdir(views_work_dir): + if fnmatch.fnmatch(file, 'TEZ{*}'): + current_version = file.lstrip("TEZ{").rstrip("}") # E.g.: TEZ{0.7.0.2.3.0.0-2154} + if self.versionCompare(current_version.replace("-", "."), last_version.replace("-", ".")) >= 0: + latest_tez_jar_version = current_version + last_version = current_version + pass + pass + pass + pass + + if latest_tez_jar_version: + tez_url = '{0}://{1}:{2}/#/main/views/TEZ/{3}/TEZ_CLUSTER_INSTANCE'.format(server_protocol, server_host, server_port, latest_tez_jar_version) + putTezProperty("tez.tez-ui.history-url.base", tez_url) + pass + + # TEZ JVM options + jvmGCParams = "-XX:+UseParallelGC" + if "ambari-server-properties" in services and "java.home" in services["ambari-server-properties"]: + # JDK8 needs different parameters + match = re.match(".*\/jdk(1\.\d+)[\-\_\.][^/]*$", services["ambari-server-properties"]["java.home"]) + if match and len(match.groups()) > 0: + # Is version >= 1.8 + versionSplits = re.split("\.", match.group(1)) + if versionSplits and len(versionSplits) > 1 and int(versionSplits[0]) > 0 and int(versionSplits[1]) > 7: + jvmGCParams = "-XX:+UseG1GC -XX:+ResizeTLAB" + putTezProperty('tez.am.launch.cmd-opts', "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA " + jvmGCParams) + putTezProperty('tez.task.launch.cmd-opts', "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA " + jvmGCParams) + + + def recommendHBASEConfigurations(self, configurations, clusterData, services, hosts): + super(HDP23StackAdvisor, self).recommendHBASEConfigurations(configurations, clusterData, services, hosts) + putHbaseSiteProperty = self.putProperty(configurations, "hbase-site", services) + putHbaseSitePropertyAttributes = self.putPropertyAttribute(configurations, "hbase-site") + putHbaseEnvProperty = self.putProperty(configurations, "hbase-env", services) + putHbaseEnvPropertyAttributes = self.putPropertyAttribute(configurations, "hbase-env") + + # bucket cache for 1.x is configured slightly differently, HBASE-11520 + threshold = 23 # 2 Gb is reserved for other offheap memory + if (int(clusterData["hbaseRam"]) > threshold): + # To enable cache - calculate values + regionserver_total_ram = int(clusterData["hbaseRam"]) * 1024 + regionserver_heap_size = 20480 + regionserver_max_direct_memory_size = regionserver_total_ram - regionserver_heap_size + hfile_block_cache_size = '0.4' + block_cache_heap = 8192 # int(regionserver_heap_size * hfile_block_cache_size) + hbase_regionserver_global_memstore_size = '0.4' + reserved_offheap_memory = 2048 + bucketcache_offheap_memory = regionserver_max_direct_memory_size - reserved_offheap_memory + hbase_bucketcache_size = bucketcache_offheap_memory + + # Set values in hbase-site + putHbaseSiteProperty('hfile.block.cache.size', hfile_block_cache_size) + putHbaseSiteProperty('hbase.regionserver.global.memstore.size', hbase_regionserver_global_memstore_size) + putHbaseSiteProperty('hbase.bucketcache.ioengine', 'offheap') + putHbaseSiteProperty('hbase.bucketcache.size', hbase_bucketcache_size) + # 2.2 stack method was called earlier, unset + putHbaseSitePropertyAttributes('hbase.bucketcache.percentage.in.combinedcache', 'delete', 'true') + + # Enable in hbase-env + putHbaseEnvProperty('hbase_max_direct_memory_size', regionserver_max_direct_memory_size) + putHbaseEnvProperty('hbase_regionserver_heapsize', regionserver_heap_size) + else: + # Disable + putHbaseSitePropertyAttributes('hbase.bucketcache.ioengine', 'delete', 'true') + putHbaseSitePropertyAttributes('hbase.bucketcache.size', 'delete', 'true') + putHbaseSitePropertyAttributes('hbase.bucketcache.percentage.in.combinedcache', 'delete', 'true') + + putHbaseEnvPropertyAttributes('hbase_max_direct_memory_size', 'delete', 'true') + + if 'hbase-env' in services['configurations'] and 'phoenix_sql_enabled' in services['configurations']['hbase-env']['properties'] and \ + 'true' == services['configurations']['hbase-env']['properties']['phoenix_sql_enabled'].lower(): + putHbaseSiteProperty("hbase.rpc.controllerfactory.class", "org.apache.hadoop.hbase.ipc.controller.ServerRpcControllerFactory") + putHbaseSiteProperty("hbase.region.server.rpc.scheduler.factory.class", "org.apache.hadoop.hbase.ipc.PhoenixRpcSchedulerFactory") + else: + putHbaseSitePropertyAttributes('hbase.region.server.rpc.scheduler.factory.class', 'delete', 'true') + + def recommendHIVEConfigurations(self, configurations, clusterData, services, hosts): + super(HDP23StackAdvisor, self).recommendHIVEConfigurations(configurations, clusterData, services, hosts) + putHiveSiteProperty = self.putProperty(configurations, "hive-site", services) + putHiveServerProperty = self.putProperty(configurations, "hiveserver2-site", services) + putHiveSitePropertyAttribute = self.putPropertyAttribute(configurations, "hive-site") + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + # hive_security_authorization == 'ranger' + if str(configurations["hive-env"]["properties"]["hive_security_authorization"]).lower() == "ranger": + putHiveServerProperty("hive.security.authorization.manager", "org.apache.ranger.authorization.hive.authorizer.RangerHiveAuthorizerFactory") + + # TEZ JVM options + if "TEZ" in self.getServiceNames(services): + jvmGCParams = "-XX:+UseParallelGC" + if "ambari-server-properties" in services and "java.home" in services["ambari-server-properties"]: + # JDK8 needs different parameters + match = re.match(".*\/jdk(1\.\d+)[\-\_\.][^/]*$", services["ambari-server-properties"]["java.home"]) + if match and len(match.groups()) > 0: + # Is version >= 1.8 + versionSplits = re.split("\.", match.group(1)) + if versionSplits and len(versionSplits) > 1 and int(versionSplits[0]) > 0 and int(versionSplits[1]) > 7: + jvmGCParams = "-XX:+UseG1GC -XX:+ResizeTLAB" + putHiveSiteProperty('hive.tez.java.opts', "-server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA " + jvmGCParams + " -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps") + + # if hive using sqla db, then we should add DataNucleus property + sqla_db_used = 'hive-env' in services['configurations'] and 'hive_database' in services['configurations']['hive-env']['properties'] and \ + services['configurations']['hive-env']['properties']['hive_database'] == 'Existing SQL Anywhere Database' + if sqla_db_used: + putHiveSiteProperty('datanucleus.rdbms.datastoreAdapterClassName','org.datanucleus.store.rdbms.adapter.SQLAnywhereAdapter') + else: + putHiveSitePropertyAttribute('datanucleus.rdbms.datastoreAdapterClassName', 'delete', 'true') + + # Atlas + hooks_property = "hive.exec.post.hooks" + atlas_hook_class = "org.apache.atlas.hive.hook.HiveHook" + if hooks_property in configurations["hive-site"]["properties"]: + hooks_value = configurations["hive-site"]["properties"][hooks_property] + else: + hooks_value = "" + + + hive_hooks = [x.strip() for x in hooks_value.split(",")] + hive_hooks = [x for x in hive_hooks if x != ""] + is_atlas_present_in_cluster = "ATLAS" in servicesList + + if is_atlas_present_in_cluster: + # Append atlas hook if not already present. + is_atlas_hook_in_config = atlas_hook_class in hive_hooks + if not is_atlas_hook_in_config: + hive_hooks.append(atlas_hook_class) + else: + # Remove the atlas hook since Atlas service is not present. + hive_hooks = [x for x in hive_hooks if x != atlas_hook_class] + + # Convert hive_hooks back to a csv, unless there are 0 elements, which should be " " + hooks_value = " " if len(hive_hooks) == 0 else ",".join(hive_hooks) + putHiveSiteProperty(hooks_property, hooks_value) + + # This is no longer used in HDP 2.5, but still needed in HDP 2.3 and 2.4 + atlas_server_host_info = self.getHostWithComponent("ATLAS", "ATLAS_SERVER", services, hosts) + if is_atlas_present_in_cluster and atlas_server_host_info: + atlas_rest_host = atlas_server_host_info['Hosts']['host_name'] + scheme = "http" + metadata_port = "21000" + atlas_server_default_https_port = "21443" + tls_enabled = "false" + if 'application-properties' in services['configurations']: + if 'atlas.enableTLS' in services['configurations']['application-properties']['properties']: + tls_enabled = services['configurations']['application-properties']['properties']['atlas.enableTLS'] + if 'atlas.server.http.port' in services['configurations']['application-properties']['properties']: + metadata_port = services['configurations']['application-properties']['properties']['atlas.server.http.port'] + if tls_enabled.lower() == "true": + scheme = "https" + if 'atlas.server.https.port' in services['configurations']['application-properties']['properties']: + metadata_port = services['configurations']['application-properties']['properties']['atlas.server.https.port'] + else: + metadata_port = atlas_server_default_https_port + putHiveSiteProperty('atlas.rest.address', '{0}://{1}:{2}'.format(scheme, atlas_rest_host, metadata_port)) + else: + putHiveSitePropertyAttribute('atlas.cluster.name', 'delete', 'true') + putHiveSitePropertyAttribute('atlas.rest.address', 'delete', 'true') + + def recommendHDFSConfigurations(self, configurations, clusterData, services, hosts): + super(HDP23StackAdvisor, self).recommendHDFSConfigurations(configurations, clusterData, services, hosts) + + putHdfsSiteProperty = self.putProperty(configurations, "hdfs-site", services) + putHdfsSitePropertyAttribute = self.putPropertyAttribute(configurations, "hdfs-site") + + if ('ranger-hdfs-plugin-properties' in services['configurations']) and ('ranger-hdfs-plugin-enabled' in services['configurations']['ranger-hdfs-plugin-properties']['properties']): + rangerPluginEnabled = '' + if 'ranger-hdfs-plugin-properties' in configurations and 'ranger-hdfs-plugin-enabled' in configurations['ranger-hdfs-plugin-properties']['properties']: + rangerPluginEnabled = configurations['ranger-hdfs-plugin-properties']['properties']['ranger-hdfs-plugin-enabled'] + elif 'ranger-hdfs-plugin-properties' in services['configurations'] and 'ranger-hdfs-plugin-enabled' in services['configurations']['ranger-hdfs-plugin-properties']['properties']: + rangerPluginEnabled = services['configurations']['ranger-hdfs-plugin-properties']['properties']['ranger-hdfs-plugin-enabled'] + + if rangerPluginEnabled and (rangerPluginEnabled.lower() == 'Yes'.lower()): + putHdfsSiteProperty("dfs.namenode.inode.attributes.provider.class",'org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer') + else: + putHdfsSitePropertyAttribute('dfs.namenode.inode.attributes.provider.class', 'delete', 'true') + else: + putHdfsSitePropertyAttribute('dfs.namenode.inode.attributes.provider.class', 'delete', 'true') + + def recommendKAFKAConfigurations(self, configurations, clusterData, services, hosts): + + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + kafka_broker = getServicesSiteProperties(services, "kafka-broker") + + security_enabled = self.isSecurityEnabled(services) + + putKafkaBrokerProperty = self.putProperty(configurations, "kafka-broker", services) + putKafkaLog4jProperty = self.putProperty(configurations, "kafka-log4j", services) + putKafkaBrokerAttributes = self.putPropertyAttribute(configurations, "kafka-broker") + + if security_enabled: + kafka_env = getServicesSiteProperties(services, "kafka-env") + kafka_user = kafka_env.get('kafka_user') if kafka_env is not None else None + + if kafka_user is not None: + kafka_super_users = kafka_broker.get('super.users') if kafka_broker is not None else None + + # kafka_super_super_users is expected to be formatted as: User:user1;User:user2 + if kafka_super_users is not None and kafka_super_users != '': + # Parse kafka_super_users to get a set of unique user names and rebuild the property value + user_names = set() + user_names.add(kafka_user) + for match in re.findall('User:([^;]*)', kafka_super_users): + user_names.add(match) + kafka_super_users = 'User:' + ";User:".join(user_names) + else: + kafka_super_users = 'User:' + kafka_user + + putKafkaBrokerProperty("super.users", kafka_super_users) + + putKafkaBrokerProperty("principal.to.local.class", "kafka.security.auth.KerberosPrincipalToLocal") + putKafkaBrokerProperty("security.inter.broker.protocol", "PLAINTEXTSASL") + putKafkaBrokerProperty("zookeeper.set.acl", "true") + + else: # not security_enabled + # remove unneeded properties + putKafkaBrokerAttributes('super.users', 'delete', 'true') + putKafkaBrokerAttributes('principal.to.local.class', 'delete', 'true') + putKafkaBrokerAttributes('security.inter.broker.protocol', 'delete', 'true') + + # Update ranger-kafka-plugin-properties/ranger-kafka-plugin-enabled to match ranger-env/ranger-kafka-plugin-enabled + if "ranger-env" in services["configurations"] \ + and "ranger-kafka-plugin-properties" in services["configurations"] \ + and "ranger-kafka-plugin-enabled" in services["configurations"]["ranger-env"]["properties"]: + putKafkaRangerPluginProperty = self.putProperty(configurations, "ranger-kafka-plugin-properties", services) + ranger_kafka_plugin_enabled = services["configurations"]["ranger-env"]["properties"]["ranger-kafka-plugin-enabled"] + putKafkaRangerPluginProperty("ranger-kafka-plugin-enabled", ranger_kafka_plugin_enabled) + + # Determine if the Ranger/Kafka Plugin is enabled + ranger_plugin_enabled = "RANGER" in servicesList + # Only if the RANGER service is installed.... + if ranger_plugin_enabled: + #setting Intial value to False, which will avoid continuing as true if below conditions are not matched, + # And set authorizer.class.name to RangerKafkaAuthorizer + ranger_plugin_enabled = False + + # If ranger-kafka-plugin-properties/ranger-kafka-plugin-enabled, + # determine if the Ranger/Kafka plug-in enabled enabled or not + if 'ranger-kafka-plugin-properties' in configurations and \ + 'ranger-kafka-plugin-enabled' in configurations['ranger-kafka-plugin-properties']['properties']: + ranger_plugin_enabled = configurations['ranger-kafka-plugin-properties']['properties']['ranger-kafka-plugin-enabled'].lower() == 'yes' + # If ranger-kafka-plugin-properties/ranger-kafka-plugin-enabled was not changed, + # determine if the Ranger/Kafka plug-in enabled enabled or not + elif 'ranger-kafka-plugin-properties' in services['configurations'] and \ + 'ranger-kafka-plugin-enabled' in services['configurations']['ranger-kafka-plugin-properties']['properties']: + ranger_plugin_enabled = services['configurations']['ranger-kafka-plugin-properties']['properties']['ranger-kafka-plugin-enabled'].lower() == 'yes' + + # Determine the value for kafka-broker/authorizer.class.name + if ranger_plugin_enabled: + # If the Ranger plugin for Kafka is enabled, set authorizer.class.name to + # "org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer" whether Kerberos is + # enabled or not. + putKafkaBrokerProperty("authorizer.class.name", 'org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer') + elif security_enabled: + putKafkaBrokerProperty("authorizer.class.name", 'kafka.security.auth.SimpleAclAuthorizer') + else: + putKafkaBrokerAttributes('authorizer.class.name', 'delete', 'true') + + #If AMS is part of Services, use the KafkaTimelineMetricsReporter for metric reporting. Default is ''. + if "AMBARI_METRICS" in servicesList: + putKafkaBrokerProperty('kafka.metrics.reporters', 'org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter') + + if ranger_plugin_enabled: + kafkaLog4jRangerLines = [{ + "name": "log4j.appender.rangerAppender", + "value": "org.apache.log4j.DailyRollingFileAppender" + }, + { + "name": "log4j.appender.rangerAppender.DatePattern", + "value": "'.'yyyy-MM-dd-HH" + }, + { + "name": "log4j.appender.rangerAppender.File", + "value": "${kafka.logs.dir}/ranger_kafka.log" + }, + { + "name": "log4j.appender.rangerAppender.layout", + "value": "org.apache.log4j.PatternLayout" + }, + { + "name": "log4j.appender.rangerAppender.layout.ConversionPattern", + "value": "%d{ISO8601} %p [%t] %C{6} (%F:%L) - %m%n" + }, + { + "name": "log4j.logger.org.apache.ranger", + "value": "INFO, rangerAppender" + }] + + # change kafka-log4j when ranger plugin is installed + if 'kafka-log4j' in services['configurations'] and 'content' in services['configurations']['kafka-log4j']['properties']: + kafkaLog4jContent = services['configurations']['kafka-log4j']['properties']['content'] + for item in range(len(kafkaLog4jRangerLines)): + if kafkaLog4jRangerLines[item]["name"] not in kafkaLog4jContent: + kafkaLog4jContent+= '\n' + kafkaLog4jRangerLines[item]["name"] + '=' + kafkaLog4jRangerLines[item]["value"] + putKafkaLog4jProperty("content",kafkaLog4jContent) + + def recommendRangerKMSConfigurations(self, configurations, clusterData, services, hosts): + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + putRangerKmsDbksProperty = self.putProperty(configurations, "dbks-site", services) + putRangerKmsProperty = self.putProperty(configurations, "kms-properties", services) + kmsEnvProperties = getSiteProperties(services['configurations'], 'kms-env') + putCoreSiteProperty = self.putProperty(configurations, "core-site", services) + putCoreSitePropertyAttribute = self.putPropertyAttribute(configurations, "core-site") + putRangerKmsAuditProperty = self.putProperty(configurations, "ranger-kms-audit", services) + + if 'kms-properties' in services['configurations'] and ('DB_FLAVOR' in services['configurations']['kms-properties']['properties']): + + rangerKmsDbFlavor = services['configurations']["kms-properties"]["properties"]["DB_FLAVOR"] + + if ('db_host' in services['configurations']['kms-properties']['properties']) and ('db_name' in services['configurations']['kms-properties']['properties']): + + rangerKmsDbHost = services['configurations']["kms-properties"]["properties"]["db_host"] + rangerKmsDbName = services['configurations']["kms-properties"]["properties"]["db_name"] + + ranger_kms_db_url_dict = { + 'MYSQL': {'ranger.ks.jpa.jdbc.driver': 'com.mysql.jdbc.Driver', + 'ranger.ks.jpa.jdbc.url': 'jdbc:mysql://' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + '/' + rangerKmsDbName}, + 'ORACLE': {'ranger.ks.jpa.jdbc.driver': 'oracle.jdbc.driver.OracleDriver', + 'ranger.ks.jpa.jdbc.url': 'jdbc:oracle:thin:@' + self.getOracleDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost, rangerKmsDbName)}, + 'POSTGRES': {'ranger.ks.jpa.jdbc.driver': 'org.postgresql.Driver', + 'ranger.ks.jpa.jdbc.url': 'jdbc:postgresql://' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + '/' + rangerKmsDbName}, + 'MSSQL': {'ranger.ks.jpa.jdbc.driver': 'com.microsoft.sqlserver.jdbc.SQLServerDriver', + 'ranger.ks.jpa.jdbc.url': 'jdbc:sqlserver://' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + ';databaseName=' + rangerKmsDbName}, + 'SQLA': {'ranger.ks.jpa.jdbc.driver': 'sap.jdbc4.sqlanywhere.IDriver', + 'ranger.ks.jpa.jdbc.url': 'jdbc:sqlanywhere:host=' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + ';database=' + rangerKmsDbName} + } + + rangerKmsDbProperties = ranger_kms_db_url_dict.get(rangerKmsDbFlavor, ranger_kms_db_url_dict['MYSQL']) + for key in rangerKmsDbProperties: + putRangerKmsDbksProperty(key, rangerKmsDbProperties.get(key)) + + if kmsEnvProperties and self.checkSiteProperties(kmsEnvProperties, 'kms_user') and 'KERBEROS' in servicesList: + kmsUser = kmsEnvProperties['kms_user'] + kmsUserOld = getOldValue(self, services, 'kms-env', 'kms_user') + putCoreSiteProperty('hadoop.proxyuser.{0}.groups'.format(kmsUser), '*') + if kmsUserOld is not None and kmsUser != kmsUserOld: + putCoreSitePropertyAttribute("hadoop.proxyuser.{0}.groups".format(kmsUserOld), 'delete', 'true') + services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.groups".format(kmsUserOld)}) + services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.groups".format(kmsUser)}) + + if "HDFS" in servicesList: + if 'core-site' in services['configurations'] and ('fs.defaultFS' in services['configurations']['core-site']['properties']): + default_fs = services['configurations']['core-site']['properties']['fs.defaultFS'] + putRangerKmsAuditProperty('xasecure.audit.destination.hdfs.dir', '{0}/{1}/{2}'.format(default_fs,'ranger','audit')) + + + def getOracleDBConnectionHostPort(self, db_type, db_host, rangerDbName): + connection_string = self.getDBConnectionHostPort(db_type, db_host) + colon_count = db_host.count(':') + if colon_count == 1 and '/' in db_host: + connection_string = "//" + connection_string + elif colon_count == 0 or colon_count == 1: + connection_string = "//" + connection_string + "/" + rangerDbName if rangerDbName else "//" + connection_string + + return connection_string + + def getDBConnectionHostPort(self, db_type, db_host): + connection_string = "" + if db_type is None or db_type == "": + return connection_string + else: + colon_count = db_host.count(':') + if colon_count == 0: + if DB_TYPE_DEFAULT_PORT_MAP.has_key(db_type): + connection_string = db_host + ":" + DB_TYPE_DEFAULT_PORT_MAP[db_type] + else: + connection_string = db_host + elif colon_count == 1: + connection_string = db_host + elif colon_count == 2: + connection_string = db_host + + return connection_string + + + def recommendRangerConfigurations(self, configurations, clusterData, services, hosts): + super(HDP23StackAdvisor, self).recommendRangerConfigurations(configurations, clusterData, services, hosts) + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + putRangerAdminProperty = self.putProperty(configurations, "ranger-admin-site", services) + putRangerEnvProperty = self.putProperty(configurations, "ranger-env", services) + putRangerUgsyncSite = self.putProperty(configurations, "ranger-ugsync-site", services) + + if 'admin-properties' in services['configurations'] and ('DB_FLAVOR' in services['configurations']['admin-properties']['properties'])\ + and ('db_host' in services['configurations']['admin-properties']['properties']) and ('db_name' in services['configurations']['admin-properties']['properties']): + + rangerDbFlavor = services['configurations']["admin-properties"]["properties"]["DB_FLAVOR"] + rangerDbHost = services['configurations']["admin-properties"]["properties"]["db_host"] + rangerDbName = services['configurations']["admin-properties"]["properties"]["db_name"] + ranger_db_url_dict = { + 'MYSQL': {'ranger.jpa.jdbc.driver': 'com.mysql.jdbc.Driver', + 'ranger.jpa.jdbc.url': 'jdbc:mysql://' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + '/' + rangerDbName}, + 'ORACLE': {'ranger.jpa.jdbc.driver': 'oracle.jdbc.driver.OracleDriver', + 'ranger.jpa.jdbc.url': 'jdbc:oracle:thin:@' + self.getOracleDBConnectionHostPort(rangerDbFlavor, rangerDbHost, rangerDbName)}, + 'POSTGRES': {'ranger.jpa.jdbc.driver': 'org.postgresql.Driver', + 'ranger.jpa.jdbc.url': 'jdbc:postgresql://' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + '/' + rangerDbName}, + 'MSSQL': {'ranger.jpa.jdbc.driver': 'com.microsoft.sqlserver.jdbc.SQLServerDriver', + 'ranger.jpa.jdbc.url': 'jdbc:sqlserver://' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + ';databaseName=' + rangerDbName}, + 'SQLA': {'ranger.jpa.jdbc.driver': 'sap.jdbc4.sqlanywhere.IDriver', + 'ranger.jpa.jdbc.url': 'jdbc:sqlanywhere:host=' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + ';database=' + rangerDbName} + } + rangerDbProperties = ranger_db_url_dict.get(rangerDbFlavor, ranger_db_url_dict['MYSQL']) + for key in rangerDbProperties: + putRangerAdminProperty(key, rangerDbProperties.get(key)) + + if 'admin-properties' in services['configurations'] and ('DB_FLAVOR' in services['configurations']['admin-properties']['properties']) \ + and ('db_host' in services['configurations']['admin-properties']['properties']): + + rangerDbFlavor = services['configurations']["admin-properties"]["properties"]["DB_FLAVOR"] + rangerDbHost = services['configurations']["admin-properties"]["properties"]["db_host"] + ranger_db_privelege_url_dict = { + 'MYSQL': {'ranger_privelege_user_jdbc_url': 'jdbc:mysql://' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost)}, + 'ORACLE': {'ranger_privelege_user_jdbc_url': 'jdbc:oracle:thin:@' + self.getOracleDBConnectionHostPort(rangerDbFlavor, rangerDbHost, None)}, + 'POSTGRES': {'ranger_privelege_user_jdbc_url': 'jdbc:postgresql://' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + '/postgres'}, + 'MSSQL': {'ranger_privelege_user_jdbc_url': 'jdbc:sqlserver://' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + ';'}, + 'SQLA': {'ranger_privelege_user_jdbc_url': 'jdbc:sqlanywhere:host=' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + ';'} + } + rangerPrivelegeDbProperties = ranger_db_privelege_url_dict.get(rangerDbFlavor, ranger_db_privelege_url_dict['MYSQL']) + for key in rangerPrivelegeDbProperties: + putRangerEnvProperty(key, rangerPrivelegeDbProperties.get(key)) + + # Recommend ldap settings based on ambari.properties configuration + if 'ambari-server-properties' in services and \ + 'ambari.ldap.isConfigured' in services['ambari-server-properties'] and \ + services['ambari-server-properties']['ambari.ldap.isConfigured'].lower() == "true": + serverProperties = services['ambari-server-properties'] + if 'authentication.ldap.baseDn' in serverProperties: + putRangerUgsyncSite('ranger.usersync.ldap.searchBase', serverProperties['authentication.ldap.baseDn']) + if 'authentication.ldap.groupMembershipAttr' in serverProperties: + putRangerUgsyncSite('ranger.usersync.group.memberattributename', serverProperties['authentication.ldap.groupMembershipAttr']) + if 'authentication.ldap.groupNamingAttr' in serverProperties: + putRangerUgsyncSite('ranger.usersync.group.nameattribute', serverProperties['authentication.ldap.groupNamingAttr']) + if 'authentication.ldap.groupObjectClass' in serverProperties: + putRangerUgsyncSite('ranger.usersync.group.objectclass', serverProperties['authentication.ldap.groupObjectClass']) + if 'authentication.ldap.managerDn' in serverProperties: + putRangerUgsyncSite('ranger.usersync.ldap.binddn', serverProperties['authentication.ldap.managerDn']) + if 'authentication.ldap.primaryUrl' in serverProperties: + ldap_protocol = 'ldap://' + if 'authentication.ldap.useSSL' in serverProperties and serverProperties['authentication.ldap.useSSL'] == 'true': + ldap_protocol = 'ldaps://' + ldapUrl = ldap_protocol + serverProperties['authentication.ldap.primaryUrl'] if serverProperties['authentication.ldap.primaryUrl'] else serverProperties['authentication.ldap.primaryUrl'] + putRangerUgsyncSite('ranger.usersync.ldap.url', ldapUrl) + if 'authentication.ldap.userObjectClass' in serverProperties: + putRangerUgsyncSite('ranger.usersync.ldap.user.objectclass', serverProperties['authentication.ldap.userObjectClass']) + if 'authentication.ldap.usernameAttribute' in serverProperties: + putRangerUgsyncSite('ranger.usersync.ldap.user.nameattribute', serverProperties['authentication.ldap.usernameAttribute']) + + + # Recommend Ranger Authentication method + authMap = { + 'org.apache.ranger.unixusersync.process.UnixUserGroupBuilder': 'UNIX', + 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder': 'LDAP' + } + + if 'ranger-ugsync-site' in services['configurations'] and 'ranger.usersync.source.impl.class' in services['configurations']["ranger-ugsync-site"]["properties"]: + rangerUserSyncClass = services['configurations']["ranger-ugsync-site"]["properties"]["ranger.usersync.source.impl.class"] + if rangerUserSyncClass in authMap: + rangerSqlConnectorProperty = authMap.get(rangerUserSyncClass) + putRangerAdminProperty('ranger.authentication.method', rangerSqlConnectorProperty) + + + if 'ranger-env' in services['configurations'] and 'is_solrCloud_enabled' in services['configurations']["ranger-env"]["properties"]: + isSolrCloudEnabled = services['configurations']["ranger-env"]["properties"]["is_solrCloud_enabled"] == "true" + else: + isSolrCloudEnabled = False + + if isSolrCloudEnabled: + zookeeper_host_port = self.getZKHostPortString(services) + ranger_audit_zk_port = '' + if zookeeper_host_port: + ranger_audit_zk_port = '{0}/{1}'.format(zookeeper_host_port, 'ranger_audits') + putRangerAdminProperty('ranger.audit.solr.zookeepers', ranger_audit_zk_port) + else: + putRangerAdminProperty('ranger.audit.solr.zookeepers', 'NONE') + + # Recommend ranger.audit.solr.zookeepers and xasecure.audit.destination.hdfs.dir + include_hdfs = "HDFS" in servicesList + if include_hdfs: + if 'core-site' in services['configurations'] and ('fs.defaultFS' in services['configurations']['core-site']['properties']): + default_fs = services['configurations']['core-site']['properties']['fs.defaultFS'] + putRangerEnvProperty('xasecure.audit.destination.hdfs.dir', '{0}/{1}/{2}'.format(default_fs,'ranger','audit')) + + # Recommend Ranger supported service's audit properties + ranger_services = [ + {'service_name': 'HDFS', 'audit_file': 'ranger-hdfs-audit'}, + {'service_name': 'YARN', 'audit_file': 'ranger-yarn-audit'}, + {'service_name': 'HBASE', 'audit_file': 'ranger-hbase-audit'}, + {'service_name': 'HIVE', 'audit_file': 'ranger-hive-audit'}, + {'service_name': 'KNOX', 'audit_file': 'ranger-knox-audit'}, + {'service_name': 'KAFKA', 'audit_file': 'ranger-kafka-audit'}, + {'service_name': 'STORM', 'audit_file': 'ranger-storm-audit'} + ] + + for item in range(len(ranger_services)): + if ranger_services[item]['service_name'] in servicesList: + component_audit_file = ranger_services[item]['audit_file'] + if component_audit_file in services["configurations"]: + ranger_audit_dict = [ + {'filename': 'ranger-env', 'configname': 'xasecure.audit.destination.db', 'target_configname': 'xasecure.audit.destination.db'}, + {'filename': 'ranger-env', 'configname': 'xasecure.audit.destination.hdfs', 'target_configname': 'xasecure.audit.destination.hdfs'}, + {'filename': 'ranger-env', 'configname': 'xasecure.audit.destination.hdfs.dir', 'target_configname': 'xasecure.audit.destination.hdfs.dir'}, + {'filename': 'ranger-env', 'configname': 'xasecure.audit.destination.solr', 'target_configname': 'xasecure.audit.destination.solr'}, + {'filename': 'ranger-admin-site', 'configname': 'ranger.audit.solr.urls', 'target_configname': 'xasecure.audit.destination.solr.urls'}, + {'filename': 'ranger-admin-site', 'configname': 'ranger.audit.solr.zookeepers', 'target_configname': 'xasecure.audit.destination.solr.zookeepers'} + ] + putRangerAuditProperty = self.putProperty(configurations, component_audit_file, services) + + for item in ranger_audit_dict: + if item['filename'] in services["configurations"] and item['configname'] in services["configurations"][item['filename']]["properties"]: + if item['filename'] in configurations and item['configname'] in configurations[item['filename']]["properties"]: + rangerAuditProperty = configurations[item['filename']]["properties"][item['configname']] + else: + rangerAuditProperty = services["configurations"][item['filename']]["properties"][item['configname']] + putRangerAuditProperty(item['target_configname'], rangerAuditProperty) + + audit_solr_flag = 'false' + audit_db_flag = 'false' + ranger_audit_source_type = 'solr' + if 'ranger-env' in services['configurations'] and 'xasecure.audit.destination.solr' in services['configurations']["ranger-env"]["properties"]: + audit_solr_flag = services['configurations']["ranger-env"]["properties"]['xasecure.audit.destination.solr'] + if 'ranger-env' in services['configurations'] and 'xasecure.audit.destination.db' in services['configurations']["ranger-env"]["properties"]: + audit_db_flag = services['configurations']["ranger-env"]["properties"]['xasecure.audit.destination.db'] + + if audit_db_flag == 'true' and audit_solr_flag == 'false': + ranger_audit_source_type = 'db' + putRangerAdminProperty('ranger.audit.source.type',ranger_audit_source_type) + + knox_host = 'localhost' + knox_port = '8443' + if 'KNOX' in servicesList: + knox_hosts = self.getComponentHostNames(services, "KNOX", "KNOX_GATEWAY") + knox_host = knox_hosts[0] + if 'gateway-site' in services['configurations'] and 'gateway.port' in services['configurations']["gateway-site"]["properties"]: + knox_port = services['configurations']["gateway-site"]["properties"]['gateway.port'] + putRangerAdminProperty('ranger.sso.providerurl', 'https://{0}:{1}/gateway/knoxsso/api/v1/websso'.format(knox_host, knox_port)) + + + def recommendYARNConfigurations(self, configurations, clusterData, services, hosts): + super(HDP23StackAdvisor, self).recommendYARNConfigurations(configurations, clusterData, services, hosts) + putYarnSiteProperty = self.putProperty(configurations, "yarn-site", services) + putYarnSitePropertyAttributes = self.putPropertyAttribute(configurations, "yarn-site") + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + + if "tez-site" not in services["configurations"]: + putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes', '') + else: + putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes', 'org.apache.tez.dag.history.logging.ats.TimelineCachePluginImpl') + + if "ranger-env" in services["configurations"] and "ranger-yarn-plugin-properties" in services["configurations"] and \ + "ranger-yarn-plugin-enabled" in services["configurations"]["ranger-env"]["properties"]: + putYarnRangerPluginProperty = self.putProperty(configurations, "ranger-yarn-plugin-properties", services) + rangerEnvYarnPluginProperty = services["configurations"]["ranger-env"]["properties"]["ranger-yarn-plugin-enabled"] + putYarnRangerPluginProperty("ranger-yarn-plugin-enabled", rangerEnvYarnPluginProperty) + rangerPluginEnabled = '' + if 'ranger-yarn-plugin-properties' in configurations and 'ranger-yarn-plugin-enabled' in configurations['ranger-yarn-plugin-properties']['properties']: + rangerPluginEnabled = configurations['ranger-yarn-plugin-properties']['properties']['ranger-yarn-plugin-enabled'] + elif 'ranger-yarn-plugin-properties' in services['configurations'] and 'ranger-yarn-plugin-enabled' in services['configurations']['ranger-yarn-plugin-properties']['properties']: + rangerPluginEnabled = services['configurations']['ranger-yarn-plugin-properties']['properties']['ranger-yarn-plugin-enabled'] + + if rangerPluginEnabled and (rangerPluginEnabled.lower() == 'Yes'.lower()): + putYarnSiteProperty('yarn.acl.enable','true') + putYarnSiteProperty('yarn.authorization-provider','org.apache.ranger.authorization.yarn.authorizer.RangerYarnAuthorizer') + else: + putYarnSitePropertyAttributes('yarn.authorization-provider', 'delete', 'true') + + if 'yarn-site' in services["configurations"] and 'yarn.resourcemanager.proxy-user-privileges.enabled' in services["configurations"]["yarn-site"]["properties"]: + if self.isSecurityEnabled(services): + # enable proxy-user privileges for secure clusters for long-running services (spark streaming etc) + putYarnSiteProperty('yarn.resourcemanager.proxy-user-privileges.enabled', 'true') + if 'RANGER_KMS' in servicesList: + # disable proxy-user privileges on secure clusters as it does not work with TDE + putYarnSiteProperty('yarn.resourcemanager.proxy-user-privileges.enabled', 'false') + else: + putYarnSiteProperty('yarn.resourcemanager.proxy-user-privileges.enabled', 'false') + + + def recommendSqoopConfigurations(self, configurations, clusterData, services, hosts): + putSqoopSiteProperty = self.putProperty(configurations, "sqoop-site", services) + + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + if "ATLAS" in servicesList: + putSqoopSiteProperty('sqoop.job.data.publish.class', 'org.apache.atlas.sqoop.hook.SqoopHook') + + def recommendStormConfigurations(self, configurations, clusterData, services, hosts): + super(HDP23StackAdvisor, self).recommendStormConfigurations(configurations, clusterData, services, hosts) + putStormStartupProperty = self.putProperty(configurations, "storm-site", services) + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + + if "storm-site" in services["configurations"]: + # atlas + notifier_plugin_property = "storm.topology.submission.notifier.plugin.class" + if notifier_plugin_property in services["configurations"]["storm-site"]["properties"] and \ + services["configurations"]["storm-site"]["properties"][notifier_plugin_property] is not None: + + notifier_plugin_value = services["configurations"]["storm-site"]["properties"][notifier_plugin_property] + else: + notifier_plugin_value = " " + + atlas_is_present = "ATLAS" in servicesList + atlas_hook_class = "org.apache.atlas.storm.hook.StormAtlasHook" + atlas_hook_is_set = atlas_hook_class in notifier_plugin_value + + if atlas_is_present and not atlas_hook_is_set: + notifier_plugin_value = atlas_hook_class if notifier_plugin_value == " " else ",".join([notifier_plugin_value, atlas_hook_class]) + + if not atlas_is_present and atlas_hook_is_set: + application_classes = [item for item in notifier_plugin_value.split(",") if item != atlas_hook_class and item != " "] + notifier_plugin_value = ",".join(application_classes) if application_classes else " " + + if notifier_plugin_value.strip() != "": + putStormStartupProperty(notifier_plugin_property, notifier_plugin_value) + else: + putStormStartupPropertyAttribute = self.putPropertyAttribute(configurations, "storm-site") + putStormStartupPropertyAttribute(notifier_plugin_property, 'delete', 'true') + + def getServiceConfigurationValidators(self): + parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators() + childValidators = { + "HDFS": {"hdfs-site": self.validateHDFSConfigurations}, + "HIVE": {"hiveserver2-site": self.validateHiveServer2Configurations, + "hive-site": self.validateHiveConfigurations}, + "HBASE": {"hbase-site": self.validateHBASEConfigurations}, + "KAFKA": {"kafka-broker": self.validateKAFKAConfigurations}, + "YARN": {"yarn-site": self.validateYARNConfigurations}, + "RANGER": {"admin-properties": self.validateRangerAdminConfigurations, + "ranger-env": self.validateRangerConfigurationsEnv} + } + self.mergeValidators(parentValidators, childValidators) + return parentValidators + + def validateHDFSConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): + parentValidationProblems = super(HDP23StackAdvisor, self).validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts) + + # We can not access property hadoop.security.authentication from the + # other config (core-site). That's why we are using another heuristics here + hdfs_site = properties + validationItems = [] #Adding Ranger Plugin logic here + ranger_plugin_properties = getSiteProperties(configurations, "ranger-hdfs-plugin-properties") + ranger_plugin_enabled = ranger_plugin_properties['ranger-hdfs-plugin-enabled'] if ranger_plugin_properties else 'No' + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + if ("RANGER" in servicesList) and (ranger_plugin_enabled.lower() == 'Yes'.lower()): + if 'dfs.namenode.inode.attributes.provider.class' not in hdfs_site or \ + hdfs_site['dfs.namenode.inode.attributes.provider.class'].lower() != 'org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer'.lower(): + validationItems.append({"config-name": 'dfs.namenode.inode.attributes.provider.class', + "item": self.getWarnItem( + "dfs.namenode.inode.attributes.provider.class needs to be set to 'org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer' if Ranger HDFS Plugin is enabled.")}) + + validationProblems = self.toConfigurationValidationProblems(validationItems, "hdfs-site") + validationProblems.extend(parentValidationProblems) + return validationProblems + + + def validateHiveConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): + parentValidationProblems = super(HDP23StackAdvisor, self).validateHiveConfigurations(properties, recommendedDefaults, configurations, services, hosts) + hive_site = properties + hive_env_properties = getSiteProperties(configurations, "hive-env") + validationItems = [] + sqla_db_used = "hive_database" in hive_env_properties and \ + hive_env_properties['hive_database'] == 'Existing SQL Anywhere Database' + prop_name = "datanucleus.rdbms.datastoreAdapterClassName" + prop_value = "org.datanucleus.store.rdbms.adapter.SQLAnywhereAdapter" + if sqla_db_used: + if not prop_name in hive_site: + validationItems.append({"config-name": prop_name, + "item": self.getWarnItem( + "If Hive using SQL Anywhere db." \ + " {0} needs to be added with value {1}".format(prop_name,prop_value))}) + elif prop_name in hive_site and hive_site[prop_name] != "org.datanucleus.store.rdbms.adapter.SQLAnywhereAdapter": + validationItems.append({"config-name": prop_name, + "item": self.getWarnItem( + "If Hive using SQL Anywhere db." \ + " {0} needs to be set to {1}".format(prop_name,prop_value))}) + + configurationValidationProblems = self.toConfigurationValidationProblems(validationItems, "hive-site") + configurationValidationProblems.extend(parentValidationProblems) + return configurationValidationProblems + + def validateHiveServer2Configurations(self, properties, recommendedDefaults, configurations, services, hosts): + super(HDP23StackAdvisor, self).validateHiveServer2Configurations(properties, recommendedDefaults, configurations, services, hosts) + hive_server2 = properties + validationItems = [] + #Adding Ranger Plugin logic here + ranger_plugin_properties = getSiteProperties(configurations, "ranger-hive-plugin-properties") + hive_env_properties = getSiteProperties(configurations, "hive-env") + ranger_plugin_enabled = 'hive_security_authorization' in hive_env_properties and hive_env_properties['hive_security_authorization'].lower() == 'ranger' + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + ##Add stack validations only if Ranger is enabled. + if ("RANGER" in servicesList): + ##Add stack validations for Ranger plugin enabled. + if ranger_plugin_enabled: + prop_name = 'hive.security.authorization.manager' + prop_val = "org.apache.ranger.authorization.hive.authorizer.RangerHiveAuthorizerFactory" + if prop_name in hive_server2 and hive_server2[prop_name] != prop_val: + validationItems.append({"config-name": prop_name, + "item": self.getWarnItem( + "If Ranger Hive Plugin is enabled."\ + " {0} under hiveserver2-site needs to be set to {1}".format(prop_name,prop_val))}) + prop_name = 'hive.security.authenticator.manager' + prop_val = "org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator" + if prop_name in hive_server2 and hive_server2[prop_name] != prop_val: + validationItems.append({"config-name": prop_name, + "item": self.getWarnItem( + "If Ranger Hive Plugin is enabled."\ + " {0} under hiveserver2-site needs to be set to {1}".format(prop_name,prop_val))}) + prop_name = 'hive.security.authorization.enabled' + prop_val = 'true' + if prop_name in hive_server2 and hive_server2[prop_name] != prop_val: + validationItems.append({"config-name": prop_name, + "item": self.getWarnItem( + "If Ranger Hive Plugin is enabled."\ + " {0} under hiveserver2-site needs to be set to {1}".format(prop_name, prop_val))}) + prop_name = 'hive.conf.restricted.list' + prop_vals = 'hive.security.authorization.enabled,hive.security.authorization.manager,hive.security.authenticator.manager'.split(',') + current_vals = [] + missing_vals = [] + if hive_server2 and prop_name in hive_server2: + current_vals = hive_server2[prop_name].split(',') + current_vals = [x.strip() for x in current_vals] + + for val in prop_vals: + if not val in current_vals: + missing_vals.append(val) + + if missing_vals: + validationItems.append({"config-name": prop_name, + "item": self.getWarnItem("If Ranger Hive Plugin is enabled."\ + " {0} under hiveserver2-site needs to contain missing value {1}".format(prop_name, ','.join(missing_vals)))}) + ##Add stack validations for Ranger plugin disabled. + elif not ranger_plugin_enabled: + prop_name = 'hive.security.authorization.manager' + prop_val = "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory" + if prop_name in hive_server2 and hive_server2[prop_name] != prop_val: + validationItems.append({"config-name": prop_name, + "item": self.getWarnItem( + "If Ranger Hive Plugin is disabled."\ + " {0} needs to be set to {1}".format(prop_name,prop_val))}) + prop_name = 'hive.security.authenticator.manager' + prop_val = "org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator" + if prop_name in hive_server2 and hive_server2[prop_name] != prop_val: + validationItems.append({"config-name": prop_name, + "item": self.getWarnItem( + "If Ranger Hive Plugin is disabled."\ + " {0} needs to be set to {1}".format(prop_name,prop_val))}) + return self.toConfigurationValidationProblems(validationItems, "hiveserver2-site") + + def validateHBASEConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): + super(HDP23StackAdvisor, self).validateHBASEConfigurations(properties, recommendedDefaults, configurations, services, hosts) + hbase_site = properties + validationItems = [] + + #Adding Ranger Plugin logic here + ranger_plugin_properties = getSiteProperties(configurations, "ranger-hbase-plugin-properties") + ranger_plugin_enabled = ranger_plugin_properties['ranger-hbase-plugin-enabled'] if ranger_plugin_properties else 'No' + prop_name = 'hbase.security.authorization' + prop_val = "true" + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + if ("RANGER" in servicesList) and (ranger_plugin_enabled.lower() == 'Yes'.lower()): + if hbase_site[prop_name] != prop_val: + validationItems.append({"config-name": prop_name, + "item": self.getWarnItem( + "If Ranger HBase Plugin is enabled."\ + "{0} needs to be set to {1}".format(prop_name,prop_val))}) + prop_name = "hbase.coprocessor.master.classes" + prop_val = "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor" + exclude_val = "org.apache.hadoop.hbase.security.access.AccessController" + if (prop_val in hbase_site[prop_name] and exclude_val not in hbase_site[prop_name]): + pass + else: + validationItems.append({"config-name": prop_name, + "item": self.getWarnItem( + "If Ranger HBase Plugin is enabled."\ + " {0} needs to contain {1} instead of {2}".format(prop_name,prop_val,exclude_val))}) + prop_name = "hbase.coprocessor.region.classes" + prop_val = "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor" + if (prop_val in hbase_site[prop_name] and exclude_val not in hbase_site[prop_name]): + pass + else: + validationItems.append({"config-name": prop_name, + "item": self.getWarnItem( + "If Ranger HBase Plugin is enabled."\ + " {0} needs to contain {1} instead of {2}".format(prop_name,prop_val,exclude_val))}) + + return self.toConfigurationValidationProblems(validationItems, "hbase-site") + + def validateKAFKAConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): + kafka_broker = properties + validationItems = [] + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + #Adding Ranger Plugin logic here + ranger_plugin_properties = getSiteProperties(configurations, "ranger-kafka-plugin-properties") + ranger_plugin_enabled = ranger_plugin_properties['ranger-kafka-plugin-enabled'] if ranger_plugin_properties else 'No' + prop_name = 'authorizer.class.name' + prop_val = "org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer" + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + if ("RANGER" in servicesList) and (ranger_plugin_enabled.lower() == 'Yes'.lower()): + if kafka_broker[prop_name] != prop_val: + validationItems.append({"config-name": prop_name, + "item": self.getWarnItem( + "If Ranger Kafka Plugin is enabled."\ + "{0} needs to be set to {1}".format(prop_name,prop_val))}) + + if 'KERBEROS' in servicesList and 'security.inter.broker.protocol' in properties: + interBrokerValue = properties['security.inter.broker.protocol'] + prop_name = 'listeners' + prop_value = properties[prop_name] + if interBrokerValue and interBrokerValue not in prop_value: + validationItems.append({"config-name": "listeners", + "item": self.getWarnItem("If kerberos is enabled "\ + "{0} need to contain {1} as one of "\ + "the protocol".format(prop_name, interBrokerValue))}) + + return self.toConfigurationValidationProblems(validationItems, "kafka-broker") + + def validateYARNConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): + yarn_site = properties + validationItems = [] + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + if 'RANGER_KMS' in servicesList and 'KERBEROS' in servicesList: + yarn_resource_proxy_enabled = yarn_site['yarn.resourcemanager.proxy-user-privileges.enabled'] + if yarn_resource_proxy_enabled.lower() == 'true': + validationItems.append({"config-name": 'yarn.resourcemanager.proxy-user-privileges.enabled', + "item": self.getWarnItem("If Ranger KMS service is installed set yarn.resourcemanager.proxy-user-privileges.enabled " \ + "property value as false under yarn-site" + )}) + + return self.toConfigurationValidationProblems(validationItems, "yarn-site") + + def isComponentUsingCardinalityForLayout(self, componentName): + return componentName in ['NFS_GATEWAY', 'PHOENIX_QUERY_SERVER', 'SPARK_THRIFTSERVER'] + + def validateRangerAdminConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): + ranger_site = properties + validationItems = [] + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + if 'RANGER' in servicesList and 'policymgr_external_url' in ranger_site: + policymgr_mgr_url = ranger_site['policymgr_external_url'] + if policymgr_mgr_url.endswith('/'): + validationItems.append({'config-name':'policymgr_external_url', + 'item':self.getWarnItem('Ranger External URL should not contain trailing slash "/"')}) + return self.toConfigurationValidationProblems(validationItems,'admin-properties') + + def validateRangerConfigurationsEnv(self, properties, recommendedDefaults, configurations, services, hosts): + parentValidationProblems = super(HDP23StackAdvisor, self).validateRangerConfigurationsEnv(properties, recommendedDefaults, configurations, services, hosts) + ranger_env_properties = properties + validationItems = [] + security_enabled = self.isSecurityEnabled(services) + + if "ranger-kafka-plugin-enabled" in ranger_env_properties and ranger_env_properties["ranger-kafka-plugin-enabled"].lower() == 'yes' and not security_enabled: + validationItems.append({"config-name": "ranger-kafka-plugin-enabled", + "item": self.getWarnItem( + "Ranger Kafka plugin should not be enabled in non-kerberos environment.")}) + + validationProblems = self.toConfigurationValidationProblems(validationItems, "ranger-env") + validationProblems.extend(parentValidationProblems) + return validationProblems +
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_24.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_24.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_24.py new file mode 100755 index 0000000..813fe42 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_24.py @@ -0,0 +1,29 @@ +#!/usr/bin/env ambari-python-wrap +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from resource_management.core.logger import Logger + +try: + from stack_advisor_23 import * +except ImportError: + #Ignore ImportError + print("stack_advisor_23 not found") + +class HDP24StackAdvisor(HDP23StackAdvisor): + pass