Repository: ambari Updated Branches: refs/heads/trunk 3e11675be -> 8a2a9fdc1
AMBARI-16255. Basic stack advisor recommendation (shard/replication) (oleewere) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8a2a9fdc Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8a2a9fdc Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8a2a9fdc Branch: refs/heads/trunk Commit: 8a2a9fdc153eee316c49e68067741d0d59339e7b Parents: 3e11675 Author: oleewere <[email protected]> Authored: Wed May 4 23:38:57 2016 +0200 Committer: oleewere <[email protected]> Committed: Wed May 4 23:40:31 2016 +0200 ---------------------------------------------------------------------- .../configuration/logsearch-properties.xml | 2 +- .../stacks/HDP/2.3/services/stack_advisor.py | 21 ++++- .../stacks/2.3/common/test_stack_advisor.py | 94 ++++++++++++++++++++ 3 files changed, 115 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8a2a9fdc/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml index dc5ebb6..172ed95 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml @@ -43,7 +43,7 @@ <value-attributes> <type>int</type> <minimum>1</minimum> - <maximum>20</maximum> + <maximum>10</maximum> <unit>int</unit> <increment-step>1</increment-step> </value-attributes> http://git-wip-us.apache.org/repos/asf/ambari/blob/8a2a9fdc/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index 7b792b9..cf0990d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -19,6 +19,7 @@ limitations under the License. import os import re import fnmatch +import math import socket DB_TYPE_DEFAULT_PORT_MAP = {"MYSQL":"3306", "ORACLE":"1521", "POSTGRES":"5432", "MSSQL":"1433", "SQLA":"2638"} @@ -171,7 +172,8 @@ class HDP23StackAdvisor(HDP22StackAdvisor): "HAWQ": self.recommendHAWQConfigurations, "FALCON": self.recommendFalconConfigurations, "STORM": self.recommendStormConfigurations, - "SQOOP": self.recommendSqoopConfigurations + "SQOOP": self.recommendSqoopConfigurations, + "LOGSEARCH" : self.recommendLogsearchConfigurations } parentRecommendConfDict.update(childRecommendConfDict) return parentRecommendConfDict @@ -834,6 +836,23 @@ class HDP23StackAdvisor(HDP22StackAdvisor): application_services_value = " " putFalconStartupProperty(application_services_property, application_services_value) + def recommendLogsearchConfigurations(self, configurations, clusterData, services, hosts): + putLogsearchProperty = self.putProperty(configurations, "logsearch-properties", services) + logsearchSolrHosts = self.getComponentHostNames(services, "LOGSEARCH", "LOGSEARCH_SOLR") + + if logsearchSolrHosts is not None and len(logsearchSolrHosts) > 0 \ + and "logsearch-properties" in services["configurations"]: + # recommend number of shard + putLogsearchAttribute = self.putPropertyAttribute(configurations, "logsearch-properties") + putLogsearchAttribute('logsearch.collection.numshards', 'minimum', len(logsearchSolrHosts)) + putLogsearchAttribute('logsearch.collection.numshards', 'maximum', 3 * len(logsearchSolrHosts)) + putLogsearchProperty("logsearch.collection.numshards", 2 * len(logsearchSolrHosts)) + # recommend replication factor + replicationReccomendFloat = math.log(len(logsearchSolrHosts), 5) + recommendedReplicationFactor = int(1 + math.floor(replicationReccomendFloat)) + putLogsearchProperty("logsearch.collection.replication.factor", recommendedReplicationFactor) + + def getServiceConfigurationValidators(self): parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators() childValidators = { http://git-wip-us.apache.org/repos/asf/ambari/blob/8a2a9fdc/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 2080c52..fcb5407 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -2881,3 +2881,97 @@ class TestHDP23StackAdvisor(TestCase): services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.7.3_23'} self.stackAdvisor.recommendSqoopConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected) + + def test_recommendLogsearchConfiguration(self): + configurations = { + "logsearch-properties": { + "properties": { + "logsearch.collection.numshards" : "5", + "logsearch.collection.replication.factor": "0" + } + } + } + + clusterData = { + "cpu": 4, + "mapMemory": 3000, + "amMemory": 2000, + "reduceMemory": 2056, + "containers": 3, + "ramPerContainer": 256 + } + expected = { + 'logsearch-properties': { + 'properties': { + "logsearch.collection.numshards" : "2", + "logsearch.collection.replication.factor": "1" + }, + "property_attributes": { + "logsearch.collection.numshards": { + "minimum": "1", + "maximum": "3" + } + } + } + } + services = { + "services": [ + { + "href": "/api/v1/stacks/HDP/versions/2.3/services/LOGSEARCH", + "StackServices": { + "service_name": "LOGSEARCH", + "service_version": "2.6.0.2.2", + "stack_name": "HDP", + "stack_version": "2.3" + }, + "components": [ + { + "StackServiceComponents": { + "advertise_version": "false", + "cardinality": "1", + "component_category": "MASTER", + "component_name": "LOGSEARCH_SOLR", + "display_name": "LogSearch Solr Instance", + "is_client": "false", + "is_master": "true", + "hostnames": [] + }, + "dependencies": [] + } + ] + }, + ], + "configurations": { + "logsearch-properties": { + "properties": { + "logsearch.collection.numshards" : "5", + "logsearch.collection.replication.factor": "0" + } + } + }, + "changed-configurations": [ ] + + } + + hosts = { + "items" : [ + { + "href" : "/api/v1/hosts/c6401.ambari.apache.org", + "Hosts" : { + "cpu_count" : 1, + "host_name" : "c6401.ambari.apache.org", + "os_arch" : "x86_64", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "c6401.ambari.apache.org", + "rack_info" : "/default-rack", + "total_mem" : 1922680 + } + } + ] + } + def return_c6401_hostname(services, service_name, component_name): + return ["c6401.ambari.apache.org"] + self.stackAdvisor.getComponentHostNames = return_c6401_hostname + self.stackAdvisor.recommendLogsearchConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations, expected)
