This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 9c908b5 AMBARI-24024 - Logsearch: service advisor does not set
recommended number of shards (#1457)
9c908b5 is described below
commit 9c908b528bece876e8fb291fda2585fdcb20e24b
Author: kasakrisz <[email protected]>
AuthorDate: Tue Jun 5 12:45:34 2018 +0200
AMBARI-24024 - Logsearch: service advisor does not set recommended number
of shards (#1457)
---
.../LOGSEARCH/0.5.0/service_advisor.py | 38 ++-----
.../LOGSEARCH/test_service_advisor.py | 123 +++++++++++++++------
2 files changed, 100 insertions(+), 61 deletions(-)
diff --git
a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py
index 059f593..2e4acca 100644
---
a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py
+++
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py
@@ -21,13 +21,6 @@ limitations under the License.
import imp
import os
import traceback
-import re
-import socket
-import fnmatch
-import math
-
-
-from resource_management.core.logger import Logger
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/')
@@ -140,34 +133,25 @@ class
LogSearchServiceAdvisor(service_advisor.ServiceAdvisor):
infraSolrHosts = self.getComponentHostNames(services,
"AMBARI_INFRA_SOLR", "INFRA_SOLR")
# if there is AMBARI_INFRA, calculate the min/max shards and
recommendations based on the number of infra solr hosts
if infraSolrHosts is not None and len(infraSolrHosts) > 0 and
"logsearch-properties" in services["configurations"]:
- replicationReccomendFloat = math.log(len(infraSolrHosts), 5)
- recommendedReplicationFactor = int(1 +
math.floor(replicationReccomendFloat))
-
recommendedMinShards = len(infraSolrHosts)
recommendedShards = 2 * len(infraSolrHosts)
- recommendedMaxShards = max(3 * len(infraSolrHosts), 5)
- # if there is no AMBARI_INFRA (i.e. external solr is used), use default
values for min/max shards and recommendations
+ recommendedMaxShards = 10 * len(infraSolrHosts)
else:
- recommendedReplicationFactor = 2
-
+ # if there is no AMBARI_INFRA (i.e. external solr is used), use
default values for min/max shards and recommendations
recommendedMinShards = 1
recommendedShards = 1
recommendedMaxShards = 100
-
putLogSearchCommonEnvProperty('logsearch_use_external_solr', 'true')
- # recommend number of shard
- putLogSearchAttribute('logsearch.collection.service.logs.numshards',
'minimum', recommendedMinShards)
- putLogSearchAttribute('logsearch.collection.service.logs.numshards',
'maximum', recommendedMaxShards)
- putLogSearchProperty("logsearch.collection.service.logs.numshards",
recommendedShards)
-
- putLogSearchAttribute('logsearch.collection.audit.logs.numshards',
'minimum', recommendedMinShards)
- putLogSearchAttribute('logsearch.collection.audit.logs.numshards',
'maximum', recommendedMaxShards)
- putLogSearchProperty("logsearch.collection.audit.logs.numshards",
recommendedShards)
- # recommend replication factor
-
putLogSearchProperty("logsearch.collection.service.logs.replication.factor",
recommendedReplicationFactor)
-
putLogSearchProperty("logsearch.collection.audit.logs.replication.factor",
recommendedReplicationFactor)
-
+ # recommend number of shard
+ putLogSearchAttribute('logsearch.collection.service.logs.numshards',
'minimum', recommendedMinShards)
+ putLogSearchAttribute('logsearch.collection.service.logs.numshards',
'maximum', recommendedMaxShards)
+ putLogSearchProperty("logsearch.collection.service.logs.numshards",
recommendedShards)
+
+ putLogSearchAttribute('logsearch.collection.audit.logs.numshards',
'minimum', recommendedMinShards)
+ putLogSearchAttribute('logsearch.collection.audit.logs.numshards',
'maximum', recommendedMaxShards)
+ putLogSearchProperty("logsearch.collection.audit.logs.numshards",
recommendedShards)
+
kerberos_authentication_enabled = self.isSecurityEnabled(services)
# if there is no kerberos enabled hide kerberor related properties
if not kerberos_authentication_enabled:
diff --git
a/ambari-server/src/test/python/common-services/LOGSEARCH/test_service_advisor.py
b/ambari-server/src/test/python/common-services/LOGSEARCH/test_service_advisor.py
index 37b02f4..c3a79a0 100644
---
a/ambari-server/src/test/python/common-services/LOGSEARCH/test_service_advisor.py
+++
b/ambari-server/src/test/python/common-services/LOGSEARCH/test_service_advisor.py
@@ -17,12 +17,9 @@ limitations under the License.
"""
import imp
-import json
import os
from unittest import TestCase
-from mock.mock import patch, MagicMock
-
class TestLOGSEARCH050ServiceAdvisor(TestCase):
@@ -41,27 +38,46 @@ class TestLOGSEARCH050ServiceAdvisor(TestCase):
with open(logserch050ServiceAdvisorPath, 'rb') as fp:
service_advisor_impl = imp.load_module('service_advisor_impl', fp,
logserch050ServiceAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+ configurations = {
+ "logsearch-properties": {
+ }
+ }
+
+ clusterData = {
+ "cpu": 4,
+ "mapMemory": 3000,
+ "amMemory": 2000,
+ "reduceMemory": 2056,
+ "containers": 3,
+ "ramPerContainer": 256
+ }
+
+ hosts = {
+ "items" : [
+ {
+ "href" : "/api/v1/hosts/c6401.ambari.apache.org",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "host_name" : "c6401.ambari.apache.org",
+ "os_arch" : "x86_64",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "c6401.ambari.apache.org",
+ "rack_info" : "/default-rack",
+ "total_mem" : 1922680
+ }
+ }
+ ]
+ }
+
def setUp(self):
serviceAdvisorClass = getattr(self.service_advisor_impl,
'LogSearchServiceAdvisor')
self.serviceAdvisor = serviceAdvisorClass()
def test_recommendLogsearchConfiguration(self):
- configurations = {
- "logsearch-properties": {
- }
- }
-
- clusterData = {
- "cpu": 4,
- "mapMemory": 3000,
- "amMemory": 2000,
- "reduceMemory": 2056,
- "containers": 3,
- "ramPerContainer": 256
- }
expected = {
'logsearch-properties': {
- 'properties': {}
+ 'properties': {'logsearch.collection.service.logs.numshards': '4',
'logsearch.collection.audit.logs.numshards': '4'}, 'property_attributes':
{'logsearch.collection.service.logs.numshards': {'minimum': '2', 'maximum':
'20'}, 'logsearch.collection.audit.logs.numshards': {'minimum': '2', 'maximum':
'20'}}
},
'logfeeder-env': {
'property_attributes': {
@@ -123,25 +139,64 @@ class TestLOGSEARCH050ServiceAdvisor(TestCase):
}
- hosts = {
- "items" : [
- {
- "href" : "/api/v1/hosts/c6401.ambari.apache.org",
- "Hosts" : {
- "cpu_count" : 1,
- "host_name" : "c6401.ambari.apache.org",
- "os_arch" : "x86_64",
- "os_type" : "centos6",
- "ph_cpu_count" : 1,
- "public_host_name" : "c6401.ambari.apache.org",
- "rack_info" : "/default-rack",
- "total_mem" : 1922680
+ def return_c6401_hostname(services, service_name, component_name):
+ return ["c6401.ambari.apache.org", "c6402.ambari.apache.org"]
+ self.serviceAdvisor.getComponentHostNames = return_c6401_hostname
+
self.serviceAdvisor.getServiceConfigurationRecommendations(self.configurations,
self.clusterData, services, self.hosts)
+ self.assertEquals(self.configurations, expected)
+
+ def test_recommendLogsearchConfigurationWhenSolrIsExternal(self):
+ expected = {
+ 'logsearch-properties': {
+ 'properties': {
+ 'logsearch.collection.service.logs.numshards': '1',
+ 'logsearch.collection.audit.logs.numshards': '1'
+ },
+ 'property_attributes': {
+ 'logsearch.collection.service.logs.numshards': {'minimum': '1',
'maximum': '100'},
+ 'logsearch.collection.audit.logs.numshards': {'minimum': '1',
'maximum': '100'}
+ }
+ },
+ 'logsearch-env': {
+ 'property_attributes': {
+ 'logsearch_external_solr_kerberos_principal': {'visible': 'false'},
+ 'logsearch_external_solr_kerberos_keytab': {'visible': 'false'}
+ }
+ },
+ 'logfeeder-env': {
+ 'property_attributes': {
+ 'logfeeder_external_solr_kerberos_keytab': {'visible': 'false'},
+ 'logfeeder_external_solr_kerberos_principal': {'visible': 'false'}
+ }
+ },
+ 'logsearch-common-env': {
+ 'properties': {
+ 'logsearch_use_external_solr': 'true',
+ 'logsearch_external_solr_kerberos_enabled': 'false'
+ },
+ 'property_attributes': {
+ 'logsearch_external_solr_kerberos_enabled': {'visible': 'false'}
+ }
+ }
+ }
+ services = {
+ "services": [],
+ "configurations": {
+ "logsearch-properties": {
+ "properties": {
+ "logsearch.collection.numshards" : "5",
+ "logsearch.collection.replication.factor": "0"
}
}
- ]
+ },
+ "changed-configurations": [ ]
+
}
def return_c6401_hostname(services, service_name, component_name):
- return ["c6401.ambari.apache.org"]
+ if service_name == "LOGSEARCH" and component_name == "LOGSEARCH_SERVER":
+ return ["c6401.ambari.apache.org"]
+ else:
+ return []
self.serviceAdvisor.getComponentHostNames = return_c6401_hostname
- self.serviceAdvisor.getServiceConfigurationRecommendations(configurations,
clusterData, services, hosts)
- self.assertEquals(configurations, expected)
+
self.serviceAdvisor.getServiceConfigurationRecommendations(self.configurations,
self.clusterData, services, self.hosts)
+ self.assertEquals(self.configurations, expected)
--
To stop receiving notification emails like this one, please contact
[email protected].
