http://git-wip-us.apache.org/repos/asf/ambari/blob/0f32765d/ambari-server/src/test/python/common-services/AMBARI_METRICS/test_service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/AMBARI_METRICS/test_service_advisor.py b/ambari-server/src/test/python/common-services/AMBARI_METRICS/test_service_advisor.py new file mode 100644 index 0000000..05254fe --- /dev/null +++ b/ambari-server/src/test/python/common-services/AMBARI_METRICS/test_service_advisor.py @@ -0,0 +1,596 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import imp +import json +import os +from unittest import TestCase + +from mock.mock import patch, MagicMock + + +class TestAMBARI_METRICS010ServiceAdvisor(TestCase): + + testDirectory = os.path.dirname(os.path.abspath(__file__)) + stack_advisor_path = os.path.join(testDirectory, '../../../../main/resources/stacks/stack_advisor.py') + with open(stack_advisor_path, 'rb') as fp: + imp.load_module('stack_advisor', fp, stack_advisor_path, ('.py', 'rb', imp.PY_SOURCE)) + + serviceAdvisorPath = '../../../../main/resources/common-services/AMBARI_METRICS/0.1.0/service_advisor.py' + ambariMetrics010ServiceAdvisorPath = os.path.join(testDirectory, serviceAdvisorPath) + with open(ambariMetrics010ServiceAdvisorPath, 'rb') as fp: + service_advisor_impl = imp.load_module('service_advisor_impl', fp, ambariMetrics010ServiceAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) + + def setUp(self): + serviceAdvisorClass = getattr(self.service_advisor_impl, 'AMBARI_METRICSServiceAdvisor') + self.serviceAdvisor = serviceAdvisorClass() + + + def test_recommendAmsConfigurations(self): + configurations = { + "hadoop-env": { + "properties": { + "hdfs_user": "hdfs", + "proxyuser_group": "users" + } + } + } + + hosts = { + "items": [ + { + "href": "/api/v1/hosts/host1", + "Hosts": { + "cpu_count": 1, + "host_name": "c6401.ambari.apache.org", + "os_arch": "x86_64", + "os_type": "centos6", + "ph_cpu_count": 1, + "public_host_name": "public.c6401.ambari.apache.org", + "rack_info": "/default-rack", + "total_mem": 2097152, + "disk_info": [{ + "size": '80000000', + "mountpoint": "/" + }] + } + }, + { + "href": "/api/v1/hosts/host2", + "Hosts": { + "cpu_count": 1, + "host_name": "c6402.ambari.apache.org", + "os_arch": "x86_64", + "os_type": "centos6", + "ph_cpu_count": 1, + "public_host_name": "public.c6402.ambari.apache.org", + "rack_info": "/default-rack", + "total_mem": 1048576, + "disk_info": [{ + "size": '800000000', + "mountpoint": "/" + }] + } + } + ]} + + + services1 = { + "services": [ + { + "StackServices": { + "service_name": "HDFS" + }, "components": [ + { + "StackServiceComponents": { + "component_name": "NAMENODE", + "hostnames": ["c6401.ambari.apache.org"] + } + } + ] + }, + { + "StackServices": { + "service_name": "AMBARI_METRICS" + }, + "components": [ + { + "StackServiceComponents": { + "component_name": "METRICS_COLLECTOR", + "hostnames": ["c6401.ambari.apache.org", "c6402.ambari.apache.org"] + } + }, { + "StackServiceComponents": { + "component_name": "METRICS_MONITOR", + "hostnames": ["c6401.ambari.apache.org", "c6402.ambari.apache.org"] + } + } + ] + }], + "configurations": configurations, + "ambari-server-properties": {"ambari-server.user":"ambari_user"} + } + + clusterData = { + "totalAvailableRam": 2048 + } + + expected = {'ams-env': {'properties': {'metrics_collector_heapsize': '512'}}, + 'ams-grafana-env': {'properties': {}, + 'property_attributes': {'metrics_grafana_password': {'visible': 'false'}}}, + 'ams-hbase-env': {'properties': {'hbase_log_dir': '/var/log/ambari-metrics-collector', + 'hbase_master_heapsize': '512', + 'hbase_master_xmn_size': '102', + 'hbase_regionserver_heapsize': '1024', + 'regionserver_xmn_size': '128'}}, + 'ams-hbase-site': {'properties': {'hbase.cluster.distributed': 'true', + 'hbase.hregion.memstore.flush.size': '134217728', + 'hbase.regionserver.global.memstore.lowerLimit': '0.3', + 'hbase.regionserver.global.memstore.upperLimit': '0.35', + 'hbase.rootdir': '/user/ams/hbase', + 'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase-tmp', + 'hbase.zookeeper.property.clientPort': '2181', + 'hfile.block.cache.size': '0.3'}}, + 'ams-site': {'properties': {'timeline.metrics.cache.commit.interval': '10', + 'timeline.metrics.cache.size': '100', + 'timeline.metrics.cluster.aggregate.splitpoints': 'master.Balancer.BalancerCluster_95th_percentile', + 'timeline.metrics.host.aggregate.splitpoints': 'master.Balancer.BalancerCluster_95th_percentile', + 'timeline.metrics.service.handler.thread.count': '20', + 'timeline.metrics.service.operation.mode': 'distributed', + 'timeline.metrics.service.watcher.disabled': 'true', + 'timeline.metrics.service.webapp.address': '0.0.0.0:6188'}}, + 'hadoop-env': {'properties': {'hdfs_user': 'hdfs', + 'proxyuser_group': 'users'}}} + + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services1, hosts) + self.assertEquals(configurations, expected) + + services1 = { + "services": [ + { + "StackServices": { + "service_name": "HDFS" + }, "components": [ + { + "StackServiceComponents": { + "component_name": "NAMENODE", + "hostnames": ["c6401.ambari.apache.org"] + } + } + ] + }, + { + "StackServices": { + "service_name": "AMBARI_METRICS" + }, + "components": [ + { + "StackServiceComponents": { + "component_name": "METRICS_COLLECTOR", + "hostnames": ["c6401.ambari.apache.org"] + } + }, { + "StackServiceComponents": { + "component_name": "METRICS_MONITOR", + "hostnames": ["c6401.ambari.apache.org", "c6402.ambari.apache.org"] + } + } + ] + }], + "configurations": configurations, + "ambari-server-properties": {"ambari-server.user":"ambari_user"} + } + expected = {'ams-env': {'properties': {'metrics_collector_heapsize': '512'}}, + 'ams-grafana-env': {'properties': {}, + 'property_attributes': {'metrics_grafana_password': {'visible': 'false'}}}, + 'ams-hbase-env': {'properties': {'hbase_log_dir': '/var/log/ambari-metrics-collector', + 'hbase_master_heapsize': '512', + 'hbase_master_xmn_size': '102', + 'hbase_regionserver_heapsize': '1024', + 'regionserver_xmn_size': '128'}}, + 'ams-hbase-site': {'properties': {'hbase.cluster.distributed': 'true', + 'hbase.hregion.memstore.flush.size': '134217728', + 'hbase.regionserver.global.memstore.lowerLimit': '0.3', + 'hbase.regionserver.global.memstore.upperLimit': '0.35', + 'hbase.rootdir': '/user/ams/hbase', + 'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase-tmp', + 'hbase.zookeeper.property.clientPort': '2181', + 'hfile.block.cache.size': '0.3', + 'phoenix.coprocessor.maxMetaDataCacheSize': '20480000'}}, + 'ams-site': {'properties': {'timeline.metrics.cache.commit.interval': '10', + 'timeline.metrics.cache.size': '100', + 'timeline.metrics.cluster.aggregate.splitpoints': 'master.Balancer.BalancerCluster_95th_percentile', + 'timeline.metrics.host.aggregate.splitpoints': 'master.Balancer.BalancerCluster_95th_percentile', + 'timeline.metrics.service.handler.thread.count': '20', + 'timeline.metrics.service.operation.mode': 'distributed', + 'timeline.metrics.service.watcher.disabled': 'true', + 'timeline.metrics.service.webapp.address': '0.0.0.0:6188'}}, + 'hadoop-env': {'properties': {'hdfs_user': 'hdfs', + 'proxyuser_group': 'users'}}} + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services1, hosts) + self.assertEquals(configurations, expected) + + + def test_validateAmsSiteConfigurations(self): + configurations = { + "hdfs-site": { + "properties": { + 'dfs.datanode.data.dir': "/hadoop/data" + } + }, + "core-site": { + "properties": { + "fs.defaultFS": "hdfs://c6401.ambari.apache.org:8020" + } + }, + "ams-site": { + "properties": { + "timeline.metrics.service.operation.mode": "embedded" + } + } + } + recommendedDefaults = { + 'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase', + 'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase', + 'hbase.cluster.distributed': 'false' + } + properties = { + 'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase', + 'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase', + 'hbase.cluster.distributed': 'false', + 'timeline.metrics.service.operation.mode' : 'embedded' + } + host1 = { + "href" : "/api/v1/hosts/host1", + "Hosts" : { + "cpu_count" : 1, + "host_name" : "host1", + "os_arch" : "x86_64", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "host1", + "rack_info" : "/default-rack", + "total_mem" : 2097152, + "disk_info": [ + { + "available": str(15<<30), # 15 GB + "type": "ext4", + "mountpoint": "/" + } + ] + } + } + host2 = { + "href" : "/api/v1/hosts/host2", + "Hosts" : { + "cpu_count" : 1, + "host_name" : "host2", + "os_arch" : "x86_64", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "host2", + "rack_info" : "/default-rack", + "total_mem" : 2097152, + "disk_info": [ + { + "available": str(15<<30), # 15 GB + "type": "ext4", + "mountpoint": "/" + } + ] + } + } + + hosts = { + "items" : [ + host1, host2 + ] + } + + services = { + "services": [ + { + "StackServices": { + "service_name": "AMBARI_METRICS" + }, + "components": [ + { + "StackServiceComponents": { + "component_name": "METRICS_COLLECTOR", + "hostnames": ["host1", "host2"] + } + }, { + "StackServiceComponents": { + "component_name": "METRICS_MONITOR", + "hostnames": ["host1", "host2"] + } + } + ] + }, + { + "StackServices": { + "service_name": "HDFS" + }, + "components": [ + { + "StackServiceComponents": { + "component_name": "DATANODE", + "hostnames": ["host1"] + } + } + ] + } + ], + "configurations": configurations + } + # only 1 partition, enough disk space, no warnings + res = self.serviceAdvisor.getAMBARI_METRICSValidator().validateAmsSiteConfigurationsFromHDP206(properties, recommendedDefaults, configurations, services, hosts) + expected = [{'config-name': 'timeline.metrics.service.operation.mode', + 'config-type': 'ams-site', + 'level': 'ERROR', + 'message': "Correct value should be 'distributed' for clusters with more then 1 Metrics collector", + 'type': 'configuration'}] + self.assertEquals(res, expected) + + + services = { + "services": [ + { + "StackServices": { + "service_name": "AMBARI_METRICS" + }, + "components": [ + { + "StackServiceComponents": { + "component_name": "METRICS_COLLECTOR", + "hostnames": ["host1"] + } + }, { + "StackServiceComponents": { + "component_name": "METRICS_MONITOR", + "hostnames": ["host1"] + } + } + ] + }, + { + "StackServices": { + "service_name": "HDFS" + }, + "components": [ + { + "StackServiceComponents": { + "component_name": "DATANODE", + "hostnames": ["host1"] + } + } + ] + } + ], + "configurations": configurations + } + res = self.serviceAdvisor.getAMBARI_METRICSValidator().validateAmsSiteConfigurationsFromHDP206(properties, recommendedDefaults, configurations, services, hosts) + expected = [] + self.assertEquals(res, expected) + + def test_validateAmsHbaseSiteConfigurations(self): + configurations = { + "hdfs-site": { + "properties": { + 'dfs.datanode.data.dir': "/hadoop/data" + } + }, + "core-site": { + "properties": { + "fs.defaultFS": "hdfs://c6401.ambari.apache.org:8020" + } + }, + "ams-site": { + "properties": { + "timeline.metrics.service.operation.mode": "embedded" + } + } + } + + recommendedDefaults = { + 'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase', + 'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase', + 'hbase.cluster.distributed': 'false' + } + properties = { + 'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase', + 'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase', + 'hbase.cluster.distributed': 'false' + } + host = { + "href" : "/api/v1/hosts/host1", + "Hosts" : { + "cpu_count" : 1, + "host_name" : "host1", + "os_arch" : "x86_64", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "host1", + "rack_info" : "/default-rack", + "total_mem" : 2097152, + "disk_info": [ + { + "available": str(15<<30), # 15 GB + "type": "ext4", + "mountpoint": "/" + } + ] + } + } + + hosts = { + "items" : [ + host + ] + } + + services = { + "services": [ + { + "StackServices": { + "service_name": "AMBARI_METRICS" + }, + "components": [ + { + "StackServiceComponents": { + "component_name": "METRICS_COLLECTOR", + "hostnames": ["host1"] + } + }, { + "StackServiceComponents": { + "component_name": "METRICS_MONITOR", + "hostnames": ["host1"] + } + } + ] + }, + { + "StackServices": { + "service_name": "HDFS" + }, + "components": [ + { + "StackServiceComponents": { + "component_name": "DATANODE", + "hostnames": ["host1"] + } + } + ] + } + ], + "configurations": configurations + } + + # only 1 partition, enough disk space, no warnings + res = self.serviceAdvisor.getAMBARI_METRICSValidator().validateAmsHbaseSiteConfigurationsFromHDP206(properties, recommendedDefaults, configurations, services, hosts) + expected = [] + self.assertEquals(res, expected) + + + # 1 partition, no enough disk space + host['Hosts']['disk_info'] = [ + { + "available" : '1', + "type" : "ext4", + "mountpoint" : "/" + } + ] + res = self.serviceAdvisor.getAMBARI_METRICSValidator().validateAmsHbaseSiteConfigurationsFromHDP206(properties, recommendedDefaults, configurations, services, hosts) + expected = [ + {'config-name': 'hbase.rootdir', + 'config-type': 'ams-hbase-site', + 'level': 'WARN', + 'message': 'Ambari Metrics disk space requirements not met. ' + '\nRecommended disk space for partition / is 10G', + 'type': 'configuration' + } + ] + self.assertEquals(res, expected) + + # 2 partitions + host['Hosts']['disk_info'] = [ + { + "available": str(15<<30), # 15 GB + "type" : "ext4", + "mountpoint" : "/grid/0" + }, + { + "available" : str(15<<30), # 15 GB + "type" : "ext4", + "mountpoint" : "/" + } + ] + recommendedDefaults = { + 'hbase.rootdir': 'file:///grid/0/var/lib/ambari-metrics-collector/hbase', + 'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase', + 'hbase.cluster.distributed': 'false' + } + properties = { + 'hbase.rootdir': 'file:///grid/0/var/lib/ambari-metrics-collector/hbase', + 'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase', + 'hbase.cluster.distributed': 'false' + } + res = self.serviceAdvisor.getAMBARI_METRICSValidator().validateAmsHbaseSiteConfigurationsFromHDP206(properties, recommendedDefaults, configurations, services, hosts) + expected = [] + self.assertEquals(res, expected) + + # dfs.dir & hbase.rootdir crosscheck + root partition + hbase.rootdir == hbase.tmp.dir warnings + properties = { + 'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase', + 'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase', + 'hbase.cluster.distributed': 'false' + } + + res = self.serviceAdvisor.getAMBARI_METRICSValidator().validateAmsHbaseSiteConfigurationsFromHDP206(properties, recommendedDefaults, configurations, services, hosts) + expected = [ + { + 'config-name': 'hbase.rootdir', + 'config-type': 'ams-hbase-site', + 'level': 'WARN', + 'message': 'It is not recommended to use root partition for hbase.rootdir', + 'type': 'configuration' + }, + { + 'config-name': 'hbase.tmp.dir', + 'config-type': 'ams-hbase-site', + 'level': 'WARN', + 'message': 'Consider not using / partition for storing metrics temporary data. ' + '/ partition is already used as hbase.rootdir to store metrics data', + 'type': 'configuration' + }, + { + 'config-name': 'hbase.rootdir', + 'config-type': 'ams-hbase-site', + 'level': 'WARN', + 'message': 'Consider not using / partition for storing metrics data. ' + '/ is already used by datanode to store HDFS data', + 'type': 'configuration' + } + ] + self.assertEquals(res, expected) + + # incorrect hbase.rootdir in distributed mode + properties = { + 'hbase.rootdir': 'file:///grid/0/var/lib/ambari-metrics-collector/hbase', + 'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase', + 'hbase.cluster.distributed': 'false' + } + configurations['ams-site']['properties']['timeline.metrics.service.operation.mode'] = 'distributed' + res = self.serviceAdvisor.getAMBARI_METRICSValidator().validateAmsHbaseSiteConfigurationsFromHDP206(properties, recommendedDefaults, configurations, services, hosts) + expected = [ + { + 'config-name': 'hbase.rootdir', + 'config-type': 'ams-hbase-site', + 'level': 'WARN', + 'message': 'In distributed mode hbase.rootdir should point to HDFS.', + 'type': 'configuration' + }, + { + 'config-name': 'hbase.cluster.distributed', + 'config-type': 'ams-hbase-site', + 'level': 'ERROR', + 'message': 'hbase.cluster.distributed property should be set to true for distributed mode', + 'type': 'configuration' + } + ] + self.assertEquals(res, expected) \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f32765d/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py index 65b23b0..38d6ecd 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py @@ -2315,214 +2315,6 @@ class TestHDP206StackAdvisor(TestCase): - def test_recommendAmsConfigurations(self): - configurations = { - "hadoop-env": { - "properties": { - "hdfs_user": "hdfs", - "proxyuser_group": "users" - } - } - } - - hosts = { - "items": [ - { - "href": "/api/v1/hosts/host1", - "Hosts": { - "cpu_count": 1, - "host_name": "c6401.ambari.apache.org", - "os_arch": "x86_64", - "os_type": "centos6", - "ph_cpu_count": 1, - "public_host_name": "public.c6401.ambari.apache.org", - "rack_info": "/default-rack", - "total_mem": 2097152, - "disk_info": [{ - "size": '80000000', - "mountpoint": "/" - }] - } - }, - { - "href": "/api/v1/hosts/host2", - "Hosts": { - "cpu_count": 1, - "host_name": "c6402.ambari.apache.org", - "os_arch": "x86_64", - "os_type": "centos6", - "ph_cpu_count": 1, - "public_host_name": "public.c6402.ambari.apache.org", - "rack_info": "/default-rack", - "total_mem": 1048576, - "disk_info": [{ - "size": '800000000', - "mountpoint": "/" - }] - } - } - ]} - - - services1 = { - "services": [ - { - "StackServices": { - "service_name": "HDFS" - }, "components": [ - { - "StackServiceComponents": { - "component_name": "NAMENODE", - "hostnames": ["c6401.ambari.apache.org"] - } - } - ] - }, - { - "StackServices": { - "service_name": "AMBARI_METRICS" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "METRICS_COLLECTOR", - "hostnames": ["c6401.ambari.apache.org", "c6402.ambari.apache.org"] - } - }, { - "StackServiceComponents": { - "component_name": "METRICS_MONITOR", - "hostnames": ["c6401.ambari.apache.org", "c6402.ambari.apache.org"] - } - } - ] - }], - "configurations": configurations, - "ambari-server-properties": {"ambari-server.user":"ambari_user"} - } - - clusterData = { - "totalAvailableRam": 2048 - } - - expected = {'ams-env': {'properties': {'metrics_collector_heapsize': '512'}}, - 'ams-grafana-env': {'properties': {}, - 'property_attributes': {'metrics_grafana_password': {'visible': 'false'}}}, - 'ams-hbase-env': {'properties': {'hbase_log_dir': '/var/log/ambari-metrics-collector', - 'hbase_master_heapsize': '512', - 'hbase_master_xmn_size': '102', - 'hbase_regionserver_heapsize': '1024', - 'regionserver_xmn_size': '128'}}, - 'ams-hbase-site': {'properties': {'hbase.cluster.distributed': 'true', - 'hbase.hregion.memstore.flush.size': '134217728', - 'hbase.regionserver.global.memstore.lowerLimit': '0.3', - 'hbase.regionserver.global.memstore.upperLimit': '0.35', - 'hbase.rootdir': '/user/ams/hbase', - 'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase-tmp', - 'hbase.zookeeper.property.clientPort': '2181', - 'hfile.block.cache.size': '0.3'}}, - 'ams-site': {'properties': {'timeline.metrics.cache.commit.interval': '10', - 'timeline.metrics.cache.size': '100', - 'timeline.metrics.cluster.aggregate.splitpoints': 'master.Balancer.BalancerCluster_95th_percentile', - 'timeline.metrics.host.aggregate.splitpoints': 'master.Balancer.BalancerCluster_95th_percentile', - 'timeline.metrics.service.handler.thread.count': '20', - 'timeline.metrics.service.operation.mode': 'distributed', - 'timeline.metrics.service.watcher.disabled': 'true', - 'timeline.metrics.service.webapp.address': '0.0.0.0:6188'}}, - 'hadoop-env': {'properties': {'hdfs_user': 'hdfs', - 'proxyuser_group': 'users'}}} - - self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services1, hosts) - self.assertEquals(configurations, expected) - - services1 = { - "services": [ - { - "StackServices": { - "service_name": "HDFS" - }, "components": [ - { - "StackServiceComponents": { - "component_name": "NAMENODE", - "hostnames": ["c6401.ambari.apache.org"] - } - } - ] - }, - { - "StackServices": { - "service_name": "AMBARI_METRICS" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "METRICS_COLLECTOR", - "hostnames": ["c6401.ambari.apache.org"] - } - }, { - "StackServiceComponents": { - "component_name": "METRICS_MONITOR", - "hostnames": ["c6401.ambari.apache.org", "c6402.ambari.apache.org"] - } - } - ] - }], - "configurations": configurations, - "ambari-server-properties": {"ambari-server.user":"ambari_user"} - } - expected = {'ams-env': {'properties': {'metrics_collector_heapsize': '512'}}, - 'ams-grafana-env': {'properties': {}, - 'property_attributes': {'metrics_grafana_password': {'visible': 'false'}}}, - 'ams-hbase-env': {'properties': {'hbase_log_dir': '/var/log/ambari-metrics-collector', - 'hbase_master_heapsize': '512', - 'hbase_master_xmn_size': '102', - 'hbase_regionserver_heapsize': '1024', - 'regionserver_xmn_size': '128'}}, - 'ams-hbase-site': {'properties': {'hbase.cluster.distributed': 'true', - 'hbase.hregion.memstore.flush.size': '134217728', - 'hbase.regionserver.global.memstore.lowerLimit': '0.3', - 'hbase.regionserver.global.memstore.upperLimit': '0.35', - 'hbase.rootdir': '/user/ams/hbase', - 'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase-tmp', - 'hbase.zookeeper.property.clientPort': '2181', - 'hfile.block.cache.size': '0.3', - 'phoenix.coprocessor.maxMetaDataCacheSize': '20480000'}}, - 'ams-site': {'properties': {'timeline.metrics.cache.commit.interval': '10', - 'timeline.metrics.cache.size': '100', - 'timeline.metrics.cluster.aggregate.splitpoints': 'master.Balancer.BalancerCluster_95th_percentile', - 'timeline.metrics.host.aggregate.splitpoints': 'master.Balancer.BalancerCluster_95th_percentile', - 'timeline.metrics.service.handler.thread.count': '20', - 'timeline.metrics.service.operation.mode': 'distributed', - 'timeline.metrics.service.watcher.disabled': 'true', - 'timeline.metrics.service.webapp.address': '0.0.0.0:6188'}}, - 'hadoop-env': {'properties': {'hdfs_user': 'hdfs', - 'proxyuser_group': 'users'}}} - self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services1, hosts) - self.assertEquals(configurations, expected) - - def test_getHostNamesWithComponent(self): - - services = { - "services": [ - { - "StackServices": { - "service_name": "SERVICE" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "COMPONENT", - "hostnames": ["host1","host2","host3"] - } - } - ] - } - ], - "configurations": {} - } - - result = self.stackAdvisor.getHostNamesWithComponent("SERVICE","COMPONENT", services) - expected = ["host1","host2","host3"] - self.assertEquals(result, expected) def test_getZKHostPortString(self): @@ -2908,374 +2700,6 @@ class TestHDP206StackAdvisor(TestCase): validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts) self.assertEquals(validation_problems, expected) - def test_validateAmsSiteConfigurations(self): - configurations = { - "hdfs-site": { - "properties": { - 'dfs.datanode.data.dir': "/hadoop/data" - } - }, - "core-site": { - "properties": { - "fs.defaultFS": "hdfs://c6401.ambari.apache.org:8020" - } - }, - "ams-site": { - "properties": { - "timeline.metrics.service.operation.mode": "embedded" - } - } - } - recommendedDefaults = { - 'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase', - 'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase', - 'hbase.cluster.distributed': 'false' - } - properties = { - 'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase', - 'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase', - 'hbase.cluster.distributed': 'false', - 'timeline.metrics.service.operation.mode' : 'embedded' - } - host1 = { - "href" : "/api/v1/hosts/host1", - "Hosts" : { - "cpu_count" : 1, - "host_name" : "host1", - "os_arch" : "x86_64", - "os_type" : "centos6", - "ph_cpu_count" : 1, - "public_host_name" : "host1", - "rack_info" : "/default-rack", - "total_mem" : 2097152, - "disk_info": [ - { - "available": str(15<<30), # 15 GB - "type": "ext4", - "mountpoint": "/" - } - ] - } - } - host2 = { - "href" : "/api/v1/hosts/host2", - "Hosts" : { - "cpu_count" : 1, - "host_name" : "host2", - "os_arch" : "x86_64", - "os_type" : "centos6", - "ph_cpu_count" : 1, - "public_host_name" : "host2", - "rack_info" : "/default-rack", - "total_mem" : 2097152, - "disk_info": [ - { - "available": str(15<<30), # 15 GB - "type": "ext4", - "mountpoint": "/" - } - ] - } - } - - hosts = { - "items" : [ - host1, host2 - ] - } - - services = { - "services": [ - { - "StackServices": { - "service_name": "AMBARI_METRICS" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "METRICS_COLLECTOR", - "hostnames": ["host1", "host2"] - } - }, { - "StackServiceComponents": { - "component_name": "METRICS_MONITOR", - "hostnames": ["host1", "host2"] - } - } - ] - }, - { - "StackServices": { - "service_name": "HDFS" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "DATANODE", - "hostnames": ["host1"] - } - } - ] - } - ], - "configurations": configurations - } - # only 1 partition, enough disk space, no warnings - res = self.stackAdvisor.validateAmsSiteConfigurations(properties, recommendedDefaults, configurations, services, hosts) - expected = [{'config-name': 'timeline.metrics.service.operation.mode', - 'config-type': 'ams-site', - 'level': 'ERROR', - 'message': "Correct value should be 'distributed' for clusters with more then 1 Metrics collector", - 'type': 'configuration'}] - self.assertEquals(res, expected) - - - services = { - "services": [ - { - "StackServices": { - "service_name": "AMBARI_METRICS" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "METRICS_COLLECTOR", - "hostnames": ["host1"] - } - }, { - "StackServiceComponents": { - "component_name": "METRICS_MONITOR", - "hostnames": ["host1"] - } - } - ] - }, - { - "StackServices": { - "service_name": "HDFS" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "DATANODE", - "hostnames": ["host1"] - } - } - ] - } - ], - "configurations": configurations - } - res = self.stackAdvisor.validateAmsSiteConfigurations(properties, recommendedDefaults, configurations, services, hosts) - expected = [] - self.assertEquals(res, expected) - - def test_validateAmsHbaseSiteConfigurations(self): - configurations = { - "hdfs-site": { - "properties": { - 'dfs.datanode.data.dir': "/hadoop/data" - } - }, - "core-site": { - "properties": { - "fs.defaultFS": "hdfs://c6401.ambari.apache.org:8020" - } - }, - "ams-site": { - "properties": { - "timeline.metrics.service.operation.mode": "embedded" - } - } - } - - recommendedDefaults = { - 'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase', - 'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase', - 'hbase.cluster.distributed': 'false' - } - properties = { - 'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase', - 'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase', - 'hbase.cluster.distributed': 'false' - } - host = { - "href" : "/api/v1/hosts/host1", - "Hosts" : { - "cpu_count" : 1, - "host_name" : "host1", - "os_arch" : "x86_64", - "os_type" : "centos6", - "ph_cpu_count" : 1, - "public_host_name" : "host1", - "rack_info" : "/default-rack", - "total_mem" : 2097152, - "disk_info": [ - { - "available": str(15<<30), # 15 GB - "type": "ext4", - "mountpoint": "/" - } - ] - } - } - - hosts = { - "items" : [ - host - ] - } - - services = { - "services": [ - { - "StackServices": { - "service_name": "AMBARI_METRICS" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "METRICS_COLLECTOR", - "hostnames": ["host1"] - } - }, { - "StackServiceComponents": { - "component_name": "METRICS_MONITOR", - "hostnames": ["host1"] - } - } - ] - }, - { - "StackServices": { - "service_name": "HDFS" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "DATANODE", - "hostnames": ["host1"] - } - } - ] - } - ], - "configurations": configurations - } - - # only 1 partition, enough disk space, no warnings - res = self.stackAdvisor.validateAmsHbaseSiteConfigurations(properties, recommendedDefaults, configurations, services, hosts) - expected = [] - self.assertEquals(res, expected) - - - # 1 partition, no enough disk space - host['Hosts']['disk_info'] = [ - { - "available" : '1', - "type" : "ext4", - "mountpoint" : "/" - } - ] - res = self.stackAdvisor.validateAmsHbaseSiteConfigurations(properties, recommendedDefaults, configurations, services, hosts) - expected = [ - {'config-name': 'hbase.rootdir', - 'config-type': 'ams-hbase-site', - 'level': 'WARN', - 'message': 'Ambari Metrics disk space requirements not met. ' - '\nRecommended disk space for partition / is 10G', - 'type': 'configuration' - } - ] - self.assertEquals(res, expected) - - # 2 partitions - host['Hosts']['disk_info'] = [ - { - "available": str(15<<30), # 15 GB - "type" : "ext4", - "mountpoint" : "/grid/0" - }, - { - "available" : str(15<<30), # 15 GB - "type" : "ext4", - "mountpoint" : "/" - } - ] - recommendedDefaults = { - 'hbase.rootdir': 'file:///grid/0/var/lib/ambari-metrics-collector/hbase', - 'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase', - 'hbase.cluster.distributed': 'false' - } - properties = { - 'hbase.rootdir': 'file:///grid/0/var/lib/ambari-metrics-collector/hbase', - 'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase', - 'hbase.cluster.distributed': 'false' - } - res = self.stackAdvisor.validateAmsHbaseSiteConfigurations(properties, recommendedDefaults, configurations, services, hosts) - expected = [] - self.assertEquals(res, expected) - - # dfs.dir & hbase.rootdir crosscheck + root partition + hbase.rootdir == hbase.tmp.dir warnings - properties = { - 'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase', - 'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase', - 'hbase.cluster.distributed': 'false' - } - - res = self.stackAdvisor.validateAmsHbaseSiteConfigurations(properties, recommendedDefaults, configurations, services, hosts) - expected = [ - { - 'config-name': 'hbase.rootdir', - 'config-type': 'ams-hbase-site', - 'level': 'WARN', - 'message': 'It is not recommended to use root partition for hbase.rootdir', - 'type': 'configuration' - }, - { - 'config-name': 'hbase.tmp.dir', - 'config-type': 'ams-hbase-site', - 'level': 'WARN', - 'message': 'Consider not using / partition for storing metrics temporary data. ' - '/ partition is already used as hbase.rootdir to store metrics data', - 'type': 'configuration' - }, - { - 'config-name': 'hbase.rootdir', - 'config-type': 'ams-hbase-site', - 'level': 'WARN', - 'message': 'Consider not using / partition for storing metrics data. ' - '/ is already used by datanode to store HDFS data', - 'type': 'configuration' - } - ] - self.assertEquals(res, expected) - - # incorrect hbase.rootdir in distributed mode - properties = { - 'hbase.rootdir': 'file:///grid/0/var/lib/ambari-metrics-collector/hbase', - 'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase', - 'hbase.cluster.distributed': 'false' - } - configurations['ams-site']['properties']['timeline.metrics.service.operation.mode'] = 'distributed' - res = self.stackAdvisor.validateAmsHbaseSiteConfigurations(properties, recommendedDefaults, configurations, services, hosts) - expected = [ - { - 'config-name': 'hbase.rootdir', - 'config-type': 'ams-hbase-site', - 'level': 'WARN', - 'message': 'In distributed mode hbase.rootdir should point to HDFS.', - 'type': 'configuration' - }, - { - 'config-name': 'hbase.cluster.distributed', - 'config-type': 'ams-hbase-site', - 'level': 'ERROR', - 'message': 'hbase.cluster.distributed property should be set to true for distributed mode', - 'type': 'configuration' - } - ] - self.assertEquals(res, expected) def test_validateStormSiteConfigurations(self): configurations = { http://git-wip-us.apache.org/repos/asf/ambari/blob/0f32765d/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index d6b572e..6f9e18e 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -2300,517 +2300,6 @@ class TestHDP22StackAdvisor(TestCase): self.stackAdvisor.recommendMapReduce2Configurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected) - def test_recommendAmsConfigurations(self): - configurations = {} - clusterData = {} - - services = { - "services": [ { - "StackServices": { - "service_name": "AMBARI_METRICS" - }, - "components": [{ - "StackServiceComponents": { - "component_name": "METRICS_COLLECTOR", - "hostnames": ["host1"] - } - - }, { - "StackServiceComponents": { - "component_name": "METRICS_MONITOR", - "hostnames": ["host1"] - } - - }] - }], - "configurations": [] - } - hosts = { - "items": [{ - "Hosts": { - "host_name": "host1", - - } - }] - } - - # 1-node cluster - expected = { - "ams-hbase-env": { - "properties": { - "hbase_master_xmn_size": "128", - "hbase_master_heapsize": "512", - "hbase_regionserver_heapsize": "512" - } - }, - "ams-grafana-env": { - "properties" : {}, - "property_attributes": { - "metrics_grafana_password": { - "visible": "false" - } - } - }, - "ams-env": { - "properties": { - "metrics_collector_heapsize": "512", - } - }, - "ams-hbase-site": { - "properties": { - "phoenix.coprocessor.maxMetaDataCacheSize": "20480000", - "hbase.regionserver.global.memstore.lowerLimit": "0.3", - "hbase.regionserver.global.memstore.upperLimit": "0.35", - "hbase.hregion.memstore.flush.size": "134217728", - "hfile.block.cache.size": "0.3", - "hbase.cluster.distributed": "false", - "hbase.rootdir": "file:///var/lib/ambari-metrics-collector/hbase", - "hbase.tmp.dir": "/var/lib/ambari-metrics-collector/hbase-tmp", - "hbase.zookeeper.property.clientPort": "61181", - } - }, - "ams-site": { - "properties": { - "timeline.metrics.cluster.aggregate.splitpoints": "mem_buffered", - "timeline.metrics.host.aggregate.splitpoints": "mem_buffered", - "timeline.metrics.service.handler.thread.count": "20", - 'timeline.metrics.service.webapp.address': '0.0.0.0:6188', - 'timeline.metrics.service.watcher.disabled': 'false', - 'timeline.metrics.cache.size': '100', - 'timeline.metrics.cache.commit.interval': '10' - } - } - } - self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts) - self.assertEquals(configurations, expected) - - # 100-nodes cluster, but still only 1 sink (METRICS_COLLECTOR) - for i in range(2, 201): - hosts['items'].extend([{ - "Hosts": { - "host_name": "host" + str(i) - } - }]) - - services['services'] = [ - { - "StackServices": { - "service_name": "AMBARI_METRICS" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "METRICS_COLLECTOR", - "hostnames": ["host1"] - } - }, - { - "StackServiceComponents": { - "component_name": "METRICS_MONITOR", - "hostnames": ["host" + str(i) for i in range(1, 201)] - } - } - ] - } - ] - - expected["ams-site"]['properties']['timeline.metrics.cache.size'] = '500' - expected["ams-site"]['properties']['timeline.metrics.cache.commit.interval'] = '7' - expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '2560' - expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '448' - expected["ams-env"]['properties']['metrics_collector_heapsize'] = '896' - - self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts) - self.assertEquals(configurations, expected) - - # 200 nodes, but with HDFS and YARN services installed on all nodes - services['services'] = [ - { - "StackServices": { - "service_name": "HDFS" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "NAMENODE", - "hostnames": ["host1"] - } - }, - { - "StackServiceComponents": { - "component_name": "DATANODE", - "hostnames": ["host" + str(i) for i in range(1, 201)] - } - } - ] - }, - { - "StackServices": { - "service_name": "YARN" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "RESOURCEMANAGER", - "hostnames": ["host1"] - } - }, - { - "StackServiceComponents": { - "component_name": "NODEMANAGER", - "hostnames": ["host" + str(i) for i in range(1, 201)] - } - } - ] - }, - { - "StackServices": { - "service_name": "AMBARI_METRICS" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "METRICS_COLLECTOR", - "hostnames": ["host1"] - } - }, - { - "StackServiceComponents": { - "component_name": "METRICS_MONITOR", - "hostnames": ["host" + str(i) for i in range(1, 201)] - } - } - ] - } - - ] - expected["ams-site"]['properties']['timeline.metrics.host.aggregate.splitpoints'] = 'dfs.FSNamesystem.FilesTotal,' \ - 'dfs.datanode.WritesFromRemoteClient,' \ - 'ipc.IPC.numCallsInReplicationQueue,' \ - 'mapred.ShuffleMetrics.ShuffleOutputsFailed,' \ - 'mem_buffered,' \ - 'read_count,' \ - 'regionserver.Server.percentFilesLocal,' \ - 'rpcdetailed.rpcdetailed.RegisterNodeManagerNumOps,' \ - 'sdisk_vdb_write_count' - expected["ams-site"]['properties']['timeline.metrics.cluster.aggregate.splitpoints'] = 'mem_total' - - expected["ams-site"]['properties']['timeline.metrics.cache.size'] = '600' - expected["ams-site"]['properties']['timeline.metrics.cache.commit.interval'] = '6' - expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '6656' - expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '1088' - expected["ams-env"]['properties']['metrics_collector_heapsize'] = '2176' - - self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts) - self.assertEquals(configurations, expected) - - # Test splitpoints, AMS embedded mode - services['changed-configurations'] = [ - { - "type": "ams-hbase-env", - "name": "hbase_master_heapsize", - "old_value": "1024" - } - ] - - services['configurations'] = { - 'core-site': {'properties': {}}, - 'ams-site': {'properties': {}}, - 'ams-hbase-site': {'properties': {}}, - 'ams-hbase-env': {'properties': {}} - } - - # Embedded mode, 512m master heapsize, no splitpoints recommended - services["configurations"]['ams-hbase-env']['properties']['hbase_master_heapsize'] = '512' - services["configurations"]['ams-hbase-site']['properties']['hbase.regionserver.global.memstore.upperLimit'] = '0.4' - services["configurations"]['ams-hbase-site']['properties']['hbase.hregion.memstore.flush.size'] = '134217728' - - expected['ams-site']['properties']['timeline.metrics.host.aggregate.splitpoints'] = 'mem_total' - expected['ams-site']['properties']['timeline.metrics.cluster.aggregate.splitpoints'] = 'mem_total' - expected['ams-hbase-env']['properties']['hbase_master_heapsize'] = '512' - - self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts) - self.assertEquals(configurations, expected) - - # Embedded mode, 4096m master heapsize, some splitpoints recommended - services["configurations"]['ams-hbase-env']['properties']['hbase_master_heapsize'] = '4096' - expected['ams-site']['properties']['timeline.metrics.host.aggregate.splitpoints'] = 'dfs.namenode.BlockReportAvgTime,' \ - 'master.AssignmentManger.Assign_mean,' \ - 'regionserver.Server.Append_median,' \ - 'rpcdetailed.rpcdetailed.client.CheckAccessNumOps' - expected['ams-site']['properties']['timeline.metrics.cluster.aggregate.splitpoints'] = 'mem_total' - expected['ams-hbase-env']['properties']['hbase_master_heapsize'] = '4096' - self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts) - self.assertEquals(configurations, expected) - - - # Embedded mode, 8192m master heapsize, more splitpoints recommended - services["configurations"]['ams-hbase-env']['properties']['hbase_master_heapsize'] = '8192' - expected['ams-hbase-env']['properties']['hbase_master_heapsize'] = '8192' - self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts) - self.assertEquals(len(configurations['ams-site']['properties']['timeline.metrics.host.aggregate.splitpoints'].split(',')), 13) - self.assertEquals(len(configurations['ams-site']['properties']['timeline.metrics.cluster.aggregate.splitpoints'].split(',')), 2) - - # Test splitpoints, AMS distributed mode - services['changed-configurations'] = [ - { - "type": "ams-hbase-env", - "name": "hbase_regionserver_heapsize", - "old_value": "512" - } - ] - services["configurations"]['ams-site']['properties']['timeline.metrics.service.operation.mode'] = 'distributed' - services["configurations"]["core-site"]["properties"]["fs.defaultFS"] = 'hdfs://host1:8020' - expected['ams-hbase-site']['properties']['hbase.cluster.distributed'] = 'true' - expected['ams-hbase-site']['properties']['hbase.rootdir'] = '/user/ams/hbase' - expected['ams-hbase-site']['properties']['hbase.zookeeper.property.clientPort'] = '2181' - expected['ams-hbase-env']['properties']['hbase_master_heapsize'] = '512' - expected['ams-hbase-site']['properties']['dfs.client.read.shortcircuit'] = 'true' - - # Distributed mode, low memory, no splitpoints recommended - expected['ams-site']['properties']['timeline.metrics.host.aggregate.splitpoints'] = 'mem_total' - expected['ams-site']['properties']['timeline.metrics.cluster.aggregate.splitpoints'] = 'mem_total' - expected['ams-hbase-env']['properties']['hbase_regionserver_heapsize'] = '6656' - expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '102' - expected['ams-hbase-env']['properties']['regionserver_xmn_size'] = '1024' - expected['ams-site']['properties']['timeline.metrics.service.watcher.disabled'] = 'true' - self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts) - self.assertEquals(configurations, expected) - - # Distributed mode, more memory, more splitpoints recommended - services["configurations"]['ams-hbase-env']['properties']['hbase_regionserver_heapsize'] = '8192' - expected['ams-hbase-env']['properties']['hbase_regionserver_heapsize'] = '8192' - self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts) - self.assertEquals(len(configurations['ams-site']['properties']['timeline.metrics.host.aggregate.splitpoints'].split(',')), 13) - self.assertEquals(len(configurations['ams-site']['properties']['timeline.metrics.cluster.aggregate.splitpoints'].split(',')), 2) - - # 2000-nodes cluster - for i in range(202, 2001): - hosts['items'].extend([{ - "Hosts": { - "host_name": "host" + str(i) - } - }]) - - services['services'] = [ - { - "StackServices": { - "service_name": "AMBARI_METRICS" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "METRICS_COLLECTOR", - "hostnames": ["host1"] - } - }, - { - "StackServiceComponents": { - "component_name": "METRICS_MONITOR", - "hostnames": ["host" + str(i) for i in range(1, 2001)] - } - } - ] - } - ] - - self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts) - self.assertEquals(configurations["ams-site"]['properties']['timeline.metrics.cache.size'], '700') - self.assertEquals(configurations["ams-site"]['properties']['timeline.metrics.cache.commit.interval'], '5') - - # 500 Nodes with HDFS, YARN, HIVE, STORM, HBASE, KAFKA, AMS - node_count = 500 - hosts = { - "items": [] - } - for i in range(1, node_count): - hosts['items'].extend([{ - "Hosts": { - "host_name": "host" + str(i) - } - }]) - - services['services'] = [ - { - "StackServices": { - "service_name": "HDFS" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "NAMENODE", - "hostnames": ["host1"] - } - } , - { - "StackServiceComponents": { - "component_name": "SECONDARY_NAMENODE", - "hostnames": ["host2"] - } - }, - { - "StackServiceComponents": { - "component_name": "DATANODE", - "hostnames": ["host" + str(i) for i in range(6, node_count + 1)] - } - } - ] - }, - { - "StackServices": { - "service_name": "HBASE" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "HBASE_MASTER", - "hostnames": ["host3"] - } - }, - { - "StackServiceComponents": { - "component_name": "HBASE_REGIONSERVER", - "hostnames": ["host" + str(i) for i in range(6, node_count + 1)] - } - } - ] - }, - { - "StackServices": { - "service_name": "YARN" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "RESOURCEMANAGER", - "hostnames": ["host4"] - } - }, - { - "StackServiceComponents": { - "component_name": "NODEMANAGER", - "hostnames": ["host" + str(i) for i in range(6, node_count + 1)] - } - } - ] - }, - { - "StackServices": { - "service_name": "HIVE" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "HIVE_METASTORE", - "hostnames": ["host3"] - } - }, - { - "StackServiceComponents": { - "component_name": "HIVE_SERVER", - "hostnames": ["host3"] - } - } - ] - }, - { - "StackServices": { - "service_name": "STORM" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "NIMBUS", - "hostnames": ["host" + str(i) for i in range(1, 6)] - } - } - ] - }, - { - "StackServices": { - "service_name": "KAFKA" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "KAFKA_BROKER", - "hostnames": ["host" + str(i) for i in range(1, 6)] - } - } - ] - }, - { - "StackServices": { - "service_name": "AMBARI_METRICS" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "METRICS_COLLECTOR", - "hostnames": ["host6"] - } - }, - { - "StackServiceComponents": { - "component_name": "METRICS_MONITOR", - "hostnames": ["host" + str(i) for i in range(6, node_count + 1)] - } - } - ] - } - ] - - services['configurations'] = { - 'core-site': {'properties': {}}, - 'ams-site': {'properties': {}}, - 'ams-hbase-site': {'properties': {}}, - 'ams-hbase-env': {'properties': {}} - } - services["configurations"]['ams-site']['properties']['timeline.metrics.service.operation.mode'] = 'distributed' - - expected['ams-hbase-site']['properties']['hbase.cluster.distributed'] = 'true' - expected['ams-hbase-site']['properties']['hbase.rootdir'] = '/user/ams/hbase' - expected['ams-hbase-site']['properties']['hbase.zookeeper.property.clientPort'] = '2181' - - expected["ams-site"]['properties']['timeline.metrics.host.aggregate.splitpoints'] = 'default.General.active_calls_api_get_all_databases,' \ - 'default.General.api_get_database_mean,' \ - 'default.General.gc.PS-MarkSweep.count,' \ - 'dfs.FsVolume.TotalDataFileIos,' \ - 'disk_free,' \ - 'jvm.JvmMetrics.MemHeapMaxM,' \ - 'kafka.network.RequestMetrics.RemoteTimeMs.request.Metadata.75percentile,' \ - 'kafka.network.RequestMetrics.ResponseQueueTimeMs.request.Update.Metadata.mean,' \ - 'load_one,master.FileSystem.MetaHlogSplitTime_75th_percentile,' \ - 'metricssystem.MetricsSystem.NumActiveSources,' \ - 'regionserver.Server.Append_95th_percentile,' \ - 'regionserver.Server.blockCacheEvictionCount,' \ - 'rpc.rpc.client.SentBytes,' \ - 'sdisk_vda1_write_bytes' - expected["ams-site"]['properties']['timeline.metrics.cluster.aggregate.splitpoints'] = 'ipc.IPC.authorizationSuccesses,' \ - 'metricssystem.MetricsSystem.PublishNumOps' - - expected["ams-site"]['properties']['timeline.metrics.cache.size'] = '700' - expected["ams-site"]['properties']['timeline.metrics.cache.commit.interval'] = '5' - expected["ams-site"]['properties']['timeline.metrics.service.resultset.fetchSize'] = '5000' - expected["ams-site"]['properties']['phoenix.query.maxGlobalMemoryPercentage'] = '30' - - expected["ams-env"]['properties']['metrics_collector_heapsize'] = '7040' - - expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '512' - expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '102' - - expected["ams-hbase-env"]['properties']['hbase_regionserver_heapsize'] = '21120' - expected["ams-hbase-env"]['properties']['regionserver_xmn_size'] = '3200' - - expected["ams-hbase-site"]['properties']['phoenix.query.maxGlobalMemoryPercentage'] = '20' - expected['ams-hbase-site']['properties']['hbase.hregion.memstore.flush.size'] = '268435456' - expected['ams-hbase-site']['properties']['hbase.regionserver.handler.count'] = '60' - expected['ams-hbase-site']['properties']['hbase.regionserver.hlog.blocksize'] = '134217728' - expected['ams-hbase-site']['properties']['hbase.regionserver.maxlogs'] = '64' - expected['ams-hbase-site']['properties']['phoenix.coprocessor.maxMetaDataCacheSize'] = '40960000' - - self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts) - self.assertEquals(configurations, expected) def test_recommendHbaseConfigurations(self): servicesList = ["HBASE"]
