AMBARI-21333 Extract Log Search part from stack advisor to a separate service advisor (mgergely)
Change-Id: Ia65deffc6d72bf0202ab4cff130f1feb22970a4d Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8ba5c3e3 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8ba5c3e3 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8ba5c3e3 Branch: refs/heads/feature-branch-AMBARI-21307 Commit: 8ba5c3e3ae582820828b345bb85da7aa935f637e Parents: aba2db1 Author: Miklos Gergely <mgerg...@hortonworks.com> Authored: Tue Jul 4 15:20:52 2017 +0200 Committer: Miklos Gergely <mgerg...@hortonworks.com> Committed: Tue Jul 4 15:20:52 2017 +0200 ---------------------------------------------------------------------- .../AMBARI_INFRA/0.1.0/service_advisor.py | 133 +++++++++++++ .../LOGSEARCH/0.5.0/service_advisor.py | 188 +++++++++++++++++++ .../stacks/HDP/2.2/services/stack_advisor.py | 66 ------- .../LOGSEARCH/test_service_advisor.py | 164 ++++++++++++++++ .../stacks/2.3/common/test_stack_advisor.py | 123 ------------ 5 files changed, 485 insertions(+), 189 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8ba5c3e3/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/service_advisor.py b/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/service_advisor.py new file mode 100644 index 0000000..9b6aace --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/service_advisor.py @@ -0,0 +1,133 @@ +#!/usr/bin/env ambari-python-wrap +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +# Python imports +import imp +import os +import traceback +import re +import socket +import fnmatch + + +from resource_management.core.logger import Logger + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/') +PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py') + +try: + with open(PARENT_FILE, 'rb') as fp: + service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE)) +except Exception as e: + traceback.print_exc() + print "Failed to load parent" + +class Ambari_InfraServiceAdvisor(service_advisor.ServiceAdvisor): + + def __init__(self, *args, **kwargs): + self.as_super = super(Ambari_InfraServiceAdvisor, self) + self.as_super.__init__(*args, **kwargs) + + # Always call these methods + self.modifyMastersWithMultipleInstances() + self.modifyCardinalitiesDict() + self.modifyHeapSizeProperties() + self.modifyNotValuableComponents() + self.modifyComponentsNotPreferableOnServer() + self.modifyComponentLayoutSchemes() + + def modifyMastersWithMultipleInstances(self): + """ + Modify the set of masters with multiple instances. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyCardinalitiesDict(self): + """ + Modify the dictionary of cardinalities. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyHeapSizeProperties(self): + """ + Modify the dictionary of heap size properties. + Must be overriden in child class. + """ + pass + + def modifyNotValuableComponents(self): + """ + Modify the set of components whose host assignment is based on other services. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyComponentsNotPreferableOnServer(self): + """ + Modify the set of components that are not preferable on the server. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyComponentLayoutSchemes(self): + """ + Modify layout scheme dictionaries for components. + The scheme dictionary basically maps the number of hosts to + host index where component should exist. + Must be overriden in child class. + """ + # Nothing to do + pass + + def getServiceComponentLayoutValidations(self, services, hosts): + """ + Get a list of errors. + Must be overriden in child class. + """ + + return [] + + def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts): + """ + Entry point. + Validate configurations for the service. Return a list of errors. + The code for this function should be the same for each Service Advisor. + """ + #Logger.info("Class: %s, Method: %s. Validating Configurations." % + # (self.__class__.__name__, inspect.stack()[0][3])) + + pass + + def getServiceConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts): + """ + Entry point. + Validate configurations for the service. Return a list of errors. + The code for this function should be the same for each Service Advisor. + """ + #Logger.info("Class: %s, Method: %s. Validating Configurations." % + # (self.__class__.__name__, inspect.stack()[0][3])) + + return [] http://git-wip-us.apache.org/repos/asf/ambari/blob/8ba5c3e3/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py new file mode 100644 index 0000000..6a61b07 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py @@ -0,0 +1,188 @@ +#!/usr/bin/env ambari-python-wrap +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +# Python imports +import imp +import os +import traceback +import re +import socket +import fnmatch +import math + + +from resource_management.core.logger import Logger + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/') +PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py') + +try: + with open(PARENT_FILE, 'rb') as fp: + service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE)) +except Exception as e: + traceback.print_exc() + print "Failed to load parent" + +class LogSearchServiceAdvisor(service_advisor.ServiceAdvisor): + + def __init__(self, *args, **kwargs): + self.as_super = super(LogSearchServiceAdvisor, self) + self.as_super.__init__(*args, **kwargs) + + # Always call these methods + self.modifyMastersWithMultipleInstances() + self.modifyCardinalitiesDict() + self.modifyHeapSizeProperties() + self.modifyNotValuableComponents() + self.modifyComponentsNotPreferableOnServer() + self.modifyComponentLayoutSchemes() + + def modifyMastersWithMultipleInstances(self): + """ + Modify the set of masters with multiple instances. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyCardinalitiesDict(self): + """ + Modify the dictionary of cardinalities. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyHeapSizeProperties(self): + """ + Modify the dictionary of heap size properties. + Must be overriden in child class. + """ + pass + + def modifyNotValuableComponents(self): + """ + Modify the set of components whose host assignment is based on other services. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyComponentsNotPreferableOnServer(self): + """ + Modify the set of components that are not preferable on the server. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyComponentLayoutSchemes(self): + """ + Modify layout scheme dictionaries for components. + The scheme dictionary basically maps the number of hosts to + host index where component should exist. + Must be overriden in child class. + """ + # Nothing to do + pass + + def getServiceComponentLayoutValidations(self, services, hosts): + """ + Get a list of errors. + Must be overriden in child class. + """ + + return [] + + def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts): + putLogSearchProperty = self.putProperty(configurations, "logsearch-properties", services) + putLogSearchAttribute = self.putPropertyAttribute(configurations, "logsearch-properties") + putLogSearchCommonEnvProperty = self.putProperty(configurations, "logsearch-common-env", services) + putLogSearchCommonEnvAttribute = self.putPropertyAttribute(configurations, "logsearch-common-env") + putLogSearchEnvAttribute = self.putPropertyAttribute(configurations, "logsearch-env") + putLogFeederEnvAttribute = self.putPropertyAttribute(configurations, "logfeeder-env") + + logSearchServerHosts = self.getComponentHostNames(services, "LOGSEARCH", "LOGSEARCH_SERVER") + if logSearchServerHosts is None or len(logSearchServerHosts) == 0: + for key in services['configurations']['logsearch-env']['properties']: + putLogSearchEnvAttribute(key, 'visible', 'false') + for key in services['configurations']['logsearch-properties']['properties']: + if key not in ['logsearch.collection.service.logs.numshards', 'logsearch.collection.audit.logs.numshards', + 'logsearch.solr.collection.service.logs', 'logsearch.solr.collection.audit.logs', + 'logsearch.service.logs.split.interval.mins', 'logsearch.audit.logs.split.interval.mins']: + putLogSearchAttribute(key, 'visible', 'false') + for key in services['configurations']['logsearch-audit_logs-solrconfig']['properties']: + self.putPropertyAttribute(configurations, "logsearch-audit_logs-solrconfig")(key, 'visible', 'false') + for key in services['configurations']['logsearch-service_logs-solrconfig']['properties']: + self.putPropertyAttribute(configurations, "logsearch-service_logs-solrconfig")(key, 'visible', 'false') + for key in services['configurations']['logsearch-log4j']['properties']: + self.putPropertyAttribute(configurations, "logsearch-log4j")(key, 'visible', 'false') + + putLogSearchProperty("logsearch.collection.service.logs.numshards", 2) + putLogSearchProperty("logsearch.collection.audit.logs.numshards", 2) + else: + infraSolrHosts = self.getComponentHostNames(services, "AMBARI_INFRA", "INFRA_SOLR") + if infraSolrHosts is not None and len(infraSolrHosts) > 0 and "logsearch-properties" in services["configurations"]: + replicationReccomendFloat = math.log(len(infraSolrHosts), 5) + recommendedReplicationFactor = int(1 + math.floor(replicationReccomendFloat)) + + recommendedMinShards = len(infraSolrHosts) + recommendedShards = 2 * len(infraSolrHosts) + recommendedMaxShards = 3 * len(infraSolrHosts) + else: + recommendedReplicationFactor = 2 + + recommendedMinShards = 1 + recommendedShards = 1 + recommendedMaxShards = 100 + + putLogSearchCommonEnvProperty('logsearch_use_external_solr', 'true') + + # recommend number of shard + putLogSearchAttribute('logsearch.collection.service.logs.numshards', 'minimum', recommendedMinShards) + putLogSearchAttribute('logsearch.collection.service.logs.numshards', 'maximum', recommendedMaxShards) + putLogSearchProperty("logsearch.collection.service.logs.numshards", recommendedShards) + + putLogSearchAttribute('logsearch.collection.audit.logs.numshards', 'minimum', recommendedMinShards) + putLogSearchAttribute('logsearch.collection.audit.logs.numshards', 'maximum', recommendedMaxShards) + putLogSearchProperty("logsearch.collection.audit.logs.numshards", recommendedShards) + # recommend replication factor + putLogSearchProperty("logsearch.collection.service.logs.replication.factor", recommendedReplicationFactor) + putLogSearchProperty("logsearch.collection.audit.logs.replication.factor", recommendedReplicationFactor) + + kerberos_authentication_enabled = self.isSecurityEnabled(services) + if not kerberos_authentication_enabled: + putLogSearchCommonEnvProperty('logsearch_external_solr_kerberos_enabled', 'false') + putLogSearchCommonEnvAttribute('logsearch_external_solr_kerberos_enabled', 'visible', 'false') + putLogSearchEnvAttribute('logsearch_external_solr_kerberos_keytab', 'visible', 'false') + putLogSearchEnvAttribute('logsearch_external_solr_kerberos_principal', 'visible', 'false') + putLogFeederEnvAttribute('logfeeder_external_solr_kerberos_keytab', 'visible', 'false') + putLogFeederEnvAttribute('logfeeder_external_solr_kerberos_principal', 'visible', 'false') + + def getServiceConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts): + """ + Entry point. + Validate configurations for the service. Return a list of errors. + The code for this function should be the same for each Service Advisor. + """ + #Logger.info("Class: %s, Method: %s. Validating Configurations." % + # (self.__class__.__name__, inspect.stack()[0][3])) + + return [] http://git-wip-us.apache.org/repos/asf/ambari/blob/8ba5c3e3/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 726514b..2dc1738 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -96,7 +96,6 @@ class HDP22StackAdvisor(HDP21StackAdvisor): "STORM": self.recommendStormConfigurations, "KNOX": self.recommendKnoxConfigurations, "RANGER": self.recommendRangerConfigurations, - "LOGSEARCH" : self.recommendLogsearchConfigurations, "SPARK": self.recommendSparkConfigurations, "KAFKA": self.recommendKafkaConfigurations, } @@ -1054,71 +1053,6 @@ class HDP22StackAdvisor(HDP21StackAdvisor): self.mergeValidators(parentValidators, childValidators) return parentValidators - def recommendLogsearchConfigurations(self, configurations, clusterData, services, hosts): - putLogsearchProperty = self.putProperty(configurations, "logsearch-properties", services) - putLogsearchAttribute = self.putPropertyAttribute(configurations, "logsearch-properties") - putLogsearchCommonEnvProperty = self.putProperty(configurations, "logsearch-common-env", services) - putLogsearchCommonEnvAttribute = self.putPropertyAttribute(configurations, "logsearch-common-env") - putLogsearchEnvAttribute = self.putPropertyAttribute(configurations, "logsearch-env") - putLogfeederEnvAttribute = self.putPropertyAttribute(configurations, "logfeeder-env") - - logSearchServerHosts = self.getComponentHostNames(services, "LOGSEARCH", "LOGSEARCH_SERVER") - if logSearchServerHosts is None or len(logSearchServerHosts) == 0: - for key in services['configurations']['logsearch-env']['properties']: - putLogsearchEnvAttribute(key, 'visible', 'false') - for key in services['configurations']['logsearch-properties']['properties']: - if key not in ['logsearch.collection.service.logs.numshards', 'logsearch.collection.audit.logs.numshards', - 'logsearch.solr.collection.service.logs', 'logsearch.solr.collection.audit.logs', - 'logsearch.service.logs.split.interval.mins', 'logsearch.audit.logs.split.interval.mins']: - putLogsearchAttribute(key, 'visible', 'false') - for key in services['configurations']['logsearch-audit_logs-solrconfig']['properties']: - self.putPropertyAttribute(configurations, "logsearch-audit_logs-solrconfig")(key, 'visible', 'false') - for key in services['configurations']['logsearch-service_logs-solrconfig']['properties']: - self.putPropertyAttribute(configurations, "logsearch-service_logs-solrconfig")(key, 'visible', 'false') - for key in services['configurations']['logsearch-log4j']['properties']: - self.putPropertyAttribute(configurations, "logsearch-log4j")(key, 'visible', 'false') - - putLogsearchProperty("logsearch.collection.service.logs.numshards", 2) - putLogsearchProperty("logsearch.collection.audit.logs.numshards", 2) - else: - infraSolrHosts = self.getComponentHostNames(services, "AMBARI_INFRA", "INFRA_SOLR") - if infraSolrHosts is not None and len(infraSolrHosts) > 0 and "logsearch-properties" in services["configurations"]: - replicationReccomendFloat = math.log(len(infraSolrHosts), 5) - recommendedReplicationFactor = int(1 + math.floor(replicationReccomendFloat)) - - recommendedMinShards = len(infraSolrHosts) - recommendedShards = 2 * len(infraSolrHosts) - recommendedMaxShards = 3 * len(infraSolrHosts) - else: - recommendedReplicationFactor = 2 - - recommendedMinShards = 1 - recommendedShards = 1 - recommendedMaxShards = 100 - - putLogsearchCommonEnvProperty('logsearch_use_external_solr', 'true') - - # recommend number of shard - putLogsearchAttribute('logsearch.collection.service.logs.numshards', 'minimum', recommendedMinShards) - putLogsearchAttribute('logsearch.collection.service.logs.numshards', 'maximum', recommendedMaxShards) - putLogsearchProperty("logsearch.collection.service.logs.numshards", recommendedShards) - - putLogsearchAttribute('logsearch.collection.audit.logs.numshards', 'minimum', recommendedMinShards) - putLogsearchAttribute('logsearch.collection.audit.logs.numshards', 'maximum', recommendedMaxShards) - putLogsearchProperty("logsearch.collection.audit.logs.numshards", recommendedShards) - # recommend replication factor - putLogsearchProperty("logsearch.collection.service.logs.replication.factor", recommendedReplicationFactor) - putLogsearchProperty("logsearch.collection.audit.logs.replication.factor", recommendedReplicationFactor) - - kerberos_authentication_enabled = self.isSecurityEnabled(services) - if not kerberos_authentication_enabled: - putLogsearchCommonEnvProperty('logsearch_external_solr_kerberos_enabled', 'false') - putLogsearchCommonEnvAttribute('logsearch_external_solr_kerberos_enabled', 'visible', 'false') - putLogsearchEnvAttribute('logsearch_external_solr_kerberos_keytab', 'visible', 'false') - putLogsearchEnvAttribute('logsearch_external_solr_kerberos_principal', 'visible', 'false') - putLogfeederEnvAttribute('logfeeder_external_solr_kerberos_keytab', 'visible', 'false') - putLogfeederEnvAttribute('logfeeder_external_solr_kerberos_principal', 'visible', 'false') - def validateTezConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.am.resource.memory.mb')}, {"config-name": 'tez.task.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.task.resource.memory.mb')}, http://git-wip-us.apache.org/repos/asf/ambari/blob/8ba5c3e3/ambari-server/src/test/python/common-services/LOGSEARCH/test_service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/LOGSEARCH/test_service_advisor.py b/ambari-server/src/test/python/common-services/LOGSEARCH/test_service_advisor.py new file mode 100644 index 0000000..3edb03c --- /dev/null +++ b/ambari-server/src/test/python/common-services/LOGSEARCH/test_service_advisor.py @@ -0,0 +1,164 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import imp +import json +import os +from unittest import TestCase + +from mock.mock import patch, MagicMock + + +class TestLOGSEARCH050ServiceAdvisor(TestCase): + + testDirectory = os.path.dirname(os.path.abspath(__file__)) + stack_advisor_path = os.path.join(testDirectory, '../../../../main/resources/stacks/stack_advisor.py') + with open(stack_advisor_path, 'rb') as fp: + imp.load_module('stack_advisor', fp, stack_advisor_path, ('.py', 'rb', imp.PY_SOURCE)) + + serviceAdvisorPath = '../../../../main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py' + logserch050ServiceAdvisorPath = os.path.join(testDirectory, serviceAdvisorPath) + with open(logserch050ServiceAdvisorPath, 'rb') as fp: + service_advisor_impl = imp.load_module('service_advisor_impl', fp, logserch050ServiceAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) + + def setUp(self): + serviceAdvisorClass = getattr(self.service_advisor_impl, 'LogSearchServiceAdvisor') + self.serviceAdvisor = serviceAdvisorClass() + + def test_recommendLogsearchConfiguration(self): + configurations = { + "logsearch-properties": { + "properties": { + "logsearch.collection.service.logs.numshards" : "5", + "logsearch.collection.service.logs.replication.factor": "0", + "logsearch.collection.audit.logs.numshards" : "5", + "logsearch.collection.audit.logs.replication.factor": "0" + } + } + } + + clusterData = { + "cpu": 4, + "mapMemory": 3000, + "amMemory": 2000, + "reduceMemory": 2056, + "containers": 3, + "ramPerContainer": 256 + } + expected = { + 'logfeeder-env': { + 'property_attributes': { + 'logfeeder_external_solr_kerberos_keytab': {'visible': 'false' + }, + 'logfeeder_external_solr_kerberos_principal': {'visible': 'false'} + } + }, + 'logsearch-common-env': { + 'properties': { + 'logsearch_external_solr_kerberos_enabled': 'false' + }, + 'property_attributes': { + 'logsearch_external_solr_kerberos_enabled': {'visible': 'false'} + } + }, + 'logsearch-env': { + 'property_attributes': { + 'logsearch_external_solr_kerberos_keytab': {'visible': 'false'}, + 'logsearch_external_solr_kerberos_principal': {'visible': 'false'} + } + }, + 'logsearch-properties': { + 'properties': { + "logsearch.collection.service.logs.numshards" : "2", + "logsearch.collection.service.logs.replication.factor": "1", + "logsearch.collection.audit.logs.numshards" : "2", + "logsearch.collection.audit.logs.replication.factor": "1" + }, + "property_attributes": { + "logsearch.collection.service.logs.numshards": { + "minimum": "1", + "maximum": "3" + }, + "logsearch.collection.audit.logs.numshards": { + "minimum": "1", + "maximum": "3" + } + } + } + } + services = { + "services": [ + { + "href": "/api/v1/stacks/HDP/versions/2.3/services/AMBARI_INFRA", + "StackServices": { + "service_name": "AMBARI_INFRA", + "service_version": "2.6.0.2.2", + "stack_name": "HDP", + "stack_version": "2.3" + }, + "components": [ + { + "StackServiceComponents": { + "advertise_version": "false", + "cardinality": "1", + "component_category": "MASTER", + "component_name": "INFRA_SOLR", + "display_name": "Infra Solr Instance", + "is_client": "false", + "is_master": "true", + "hostnames": [] + }, + "dependencies": [] + } + ] + }, + ], + "configurations": { + "logsearch-properties": { + "properties": { + "logsearch.collection.numshards" : "5", + "logsearch.collection.replication.factor": "0" + } + } + }, + "changed-configurations": [ ] + + } + + hosts = { + "items" : [ + { + "href" : "/api/v1/hosts/c6401.ambari.apache.org", + "Hosts" : { + "cpu_count" : 1, + "host_name" : "c6401.ambari.apache.org", + "os_arch" : "x86_64", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "c6401.ambari.apache.org", + "rack_info" : "/default-rack", + "total_mem" : 1922680 + } + } + ] + } + def return_c6401_hostname(services, service_name, component_name): + return ["c6401.ambari.apache.org"] + self.serviceAdvisor.getComponentHostNames = return_c6401_hostname + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts) + self.assertEquals(configurations, expected) http://git-wip-us.apache.org/repos/asf/ambari/blob/8ba5c3e3/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index c28ee2a..9e882d2 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -1948,129 +1948,6 @@ class TestHDP23StackAdvisor(TestCase): self.stackAdvisor.recommendSqoopConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected) - def test_recommendLogsearchConfiguration(self): - configurations = { - "logsearch-properties": { - "properties": { - "logsearch.collection.service.logs.numshards" : "5", - "logsearch.collection.service.logs.replication.factor": "0", - "logsearch.collection.audit.logs.numshards" : "5", - "logsearch.collection.audit.logs.replication.factor": "0" - } - } - } - - clusterData = { - "cpu": 4, - "mapMemory": 3000, - "amMemory": 2000, - "reduceMemory": 2056, - "containers": 3, - "ramPerContainer": 256 - } - expected = { - 'logfeeder-env': { - 'property_attributes': { - 'logfeeder_external_solr_kerberos_keytab': {'visible': 'false' - }, - 'logfeeder_external_solr_kerberos_principal': {'visible': 'false'} - } - }, - 'logsearch-common-env': { - 'properties': { - 'logsearch_external_solr_kerberos_enabled': 'false' - }, - 'property_attributes': { - 'logsearch_external_solr_kerberos_enabled': {'visible': 'false'} - } - }, - 'logsearch-env': { - 'property_attributes': { - 'logsearch_external_solr_kerberos_keytab': {'visible': 'false'}, - 'logsearch_external_solr_kerberos_principal': {'visible': 'false'} - } - }, - 'logsearch-properties': { - 'properties': { - "logsearch.collection.service.logs.numshards" : "2", - "logsearch.collection.service.logs.replication.factor": "1", - "logsearch.collection.audit.logs.numshards" : "2", - "logsearch.collection.audit.logs.replication.factor": "1" - }, - "property_attributes": { - "logsearch.collection.service.logs.numshards": { - "minimum": "1", - "maximum": "3" - }, - "logsearch.collection.audit.logs.numshards": { - "minimum": "1", - "maximum": "3" - } - } - } - } - services = { - "services": [ - { - "href": "/api/v1/stacks/HDP/versions/2.3/services/AMBARI_INFRA", - "StackServices": { - "service_name": "AMBARI_INFRA", - "service_version": "2.6.0.2.2", - "stack_name": "HDP", - "stack_version": "2.3" - }, - "components": [ - { - "StackServiceComponents": { - "advertise_version": "false", - "cardinality": "1", - "component_category": "MASTER", - "component_name": "INFRA_SOLR", - "display_name": "Infra Solr Instance", - "is_client": "false", - "is_master": "true", - "hostnames": [] - }, - "dependencies": [] - } - ] - }, - ], - "configurations": { - "logsearch-properties": { - "properties": { - "logsearch.collection.numshards" : "5", - "logsearch.collection.replication.factor": "0" - } - } - }, - "changed-configurations": [ ] - - } - - hosts = { - "items" : [ - { - "href" : "/api/v1/hosts/c6401.ambari.apache.org", - "Hosts" : { - "cpu_count" : 1, - "host_name" : "c6401.ambari.apache.org", - "os_arch" : "x86_64", - "os_type" : "centos6", - "ph_cpu_count" : 1, - "public_host_name" : "c6401.ambari.apache.org", - "rack_info" : "/default-rack", - "total_mem" : 1922680 - } - } - ] - } - def return_c6401_hostname(services, service_name, component_name): - return ["c6401.ambari.apache.org"] - self.stackAdvisor.getComponentHostNames = return_c6401_hostname - self.stackAdvisor.recommendLogsearchConfigurations(configurations, clusterData, services, hosts) - self.assertEquals(configurations, expected) - def test_validateRangerConfigurationsEnv(self): properties = { "ranger-kafka-plugin-enabled": "Yes",
