http://git-wip-us.apache.org/repos/asf/ambari/blob/11cd9237/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py new file mode 100644 index 0000000..ee1d61c --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from resource_management.libraries.script.script import Script + +config = Script.get_config() + +druid_pid_dir = config['configurations']['druid-env']['druid_pid_dir']
http://git-wip-us.apache.org/repos/asf/ambari/blob/11cd9237/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json new file mode 100644 index 0000000..c68b9b9 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json @@ -0,0 +1,37 @@ +{ + "name": "default", + "description": "default quick links configuration", + "configuration": { + "protocol": { + "type": "HTTP_ONLY" + }, + "links": [ + { + "name": "coordinator_console", + "label": "Druid Coordinator Console", + "component_name": "DRUID_COORDINATOR", + "requires_user_name": "false", + "url": "%@://%@:%@", + "port": { + "http_property": "druid.port", + "http_default_port": "8081", + "regex": "^(\\d+)$", + "site": "druid-coordinator" + } + }, + { + "name": "overlord_console", + "label": "Druid Overlord Console", + "component_name": "DRUID_OVERLORD", + "requires_user_name": "false", + "url": "%@://%@:%@", + "port": { + "http_property": "druid.port", + "http_default_port": "8090", + "regex": "^(\\d+)$", + "site": "druid-overlord" + } + } + ] + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/11cd9237/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json new file mode 100644 index 0000000..f494594 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json @@ -0,0 +1,120 @@ +{ + "name": "default", + "description": "Default theme for Druid service", + "configuration": { + "layouts": [ + { + "name": "default", + "tabs": [ + { + "name": "metadata_storage", + "display-name": "META DATA STORAGE CONFIG", + "layout": { + "tab-columns": "1", + "tab-rows": "1", + "sections": [ + { + "name": "section-metadata-storage", + "display-name": "", + "row-index": "0", + "column-index": "0", + "row-span": "2", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-metadata-storage-row1-col1", + "display-name": "META DATA STORAGE", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + } + ] + } + } + ] + } + ], + "placement": { + "configuration-layout": "default", + "configs": [ + { + "config": "druid-common/database_name", + "subsection-name": "subsection-metadata-storage-row1-col1" + }, + { + "config": "druid-common/druid.metadata.storage.type", + "subsection-name": "subsection-metadata-storage-row1-col1" + }, + { + "config": "druid-common/druid.metadata.storage.connector.user", + "subsection-name": "subsection-metadata-storage-row1-col1" + }, + { + "config": "druid-common/druid.metadata.storage.connector.password", + "subsection-name": "subsection-metadata-storage-row1-col1" + }, + { + "config": "druid-common/metastore_hostname", + "subsection-name": "subsection-metadata-storage-row1-col1" + }, + { + "config": "druid-common/druid.metadata.storage.connector.port", + "subsection-name": "subsection-metadata-storage-row1-col1" + }, + { + "config": "druid-common/druid.metadata.storage.connector.connectURI", + "subsection-name": "subsection-metadata-storage-row1-col1" + } + ] + }, + "widgets": [ + { + "config": "druid-common/database_name", + "widget": { + "type": "text-field" + } + }, + { + "config": "druid-common/druid.metadata.storage.type", + "widget": { + "type": "combo" + } + }, + { + "config": "druid-common/druid.metadata.storage.connector.user", + "widget": { + "type": "text-field" + } + }, + { + "config": "druid-common/druid.metadata.storage.connector.password", + "widget": { + "type": "password" + } + }, + { + "config": "druid-common/metastore_hostname", + "widget": { + "type": "text-field" + } + }, + { + "config": "druid-common/druid.metadata.storage.connector.port", + "widget": { + "type": "text-field" + } + }, + { + "config": "druid-common/druid.metadata.storage.connector.connectURI", + "widget": { + "type": "text-field" + } + } + ] + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/11cd9237/ambari-server/src/main/resources/stacks/HDP/2.6/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.6/role_command_order.json new file mode 100644 index 0000000..66b6ed2 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/role_command_order.json @@ -0,0 +1,19 @@ +{ + "_comment" : "Record format:", + "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]", + "general_deps" : { + "_comment" : "dependencies for all cases", + "DRUID_HISTORICAL-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"], + "DRUID_OVERLORD-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"], + "DRUID_MIDDLEMANAGER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"], + "DRUID_BROKER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"], + "DRUID_ROUTER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"], + "DRUID_COORDINATOR-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"], + "DRUID_OVERLORD-RESTART" : ["DRUID_HISTORICAL-RESTART"], + "DRUID_MIDDLEMANAGER-RESTART" : ["DRUID_OVERLORD-RESTART"], + "DRUID_BROKER-RESTART" : ["DRUID_MIDDLEMANAGER-RESTART"], + "DRUID_ROUTER-RESTART" : ["DRUID_BROKER-RESTART"], + "DRUID_COORDINATOR-RESTART" : ["DRUID_ROUTER-RESTART"], + "DRUID_SERVICE_CHECK-SERVICE_CHECK" : ["DRUID_HISTORICAL-START", "DRUID_COORDINATOR-START", "DRUID_OVERLORD-START", "DRUID_MIDDLEMANAGER-START", "DRUID_BROKER-START", "DRUID_ROUTER-START"] + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/11cd9237/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/kerberos.json new file mode 100644 index 0000000..1661285 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/kerberos.json @@ -0,0 +1,78 @@ +{ + "services": [ + { + "name": "DRUID", + "identities": [ + { + "name": "/spnego" + }, + { + "name": "druid", + "principal": { + "value": "${druid-env/druid_user}@${realm}", + "type": "user", + "configuration": "druid-common/druid.hadoop.security.kerberos.principal", + "local_username": "${druid-env/druid_user}" + }, + "keytab": { + "file": "${keytab_dir}/druid.headless.keytab", + "owner": { + "name": "${druid-env/druid_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "r" + }, + "configuration": "druid-common/druid.hadoop.security.kerberos.keytab" + } + }, + { + "name": "/smokeuser" + } + ], + "components": [ + { + "name": "DRUID_HISTORICAL", + "identities": [ + { + "name": "/druid" + } + ] + }, + { + "name": "DRUID_BROKER", + "identities": [ + { + "name": "/druid" + } + ] + }, + { + "name": "DRUID_OVERLORD", + "identities": [ + { + "name": "/druid" + } + ] + }, + { + "name": "DRUID_COORDINATOR", + "identities": [ + { + "name": "/druid" + } + ] + }, + { + "name": "DRUID_MIDDLEMANAGER", + "identities": [ + { + "name": "/druid" + } + ] + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/ambari/blob/11cd9237/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/metainfo.xml new file mode 100644 index 0000000..49d09b1 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/metainfo.xml @@ -0,0 +1,27 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>DRUID</name> + <version>0.9.2.2.6</version> + <extends>common-services/DRUID/0.9.2</extends> + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/11cd9237/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py index 1f722dc..49dd086 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py @@ -16,6 +16,191 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +from resource_management.core.logger import Logger +import json +from resource_management.libraries.functions import format + class HDP26StackAdvisor(HDP25StackAdvisor): - pass \ No newline at end of file + def __init__(self): + super(HDP26StackAdvisor, self).__init__() + Logger.initialize_logger() + + def getServiceConfigurationRecommenderDict(self): + parentRecommendConfDict = super(HDP26StackAdvisor, self).getServiceConfigurationRecommenderDict() + childRecommendConfDict = { + "DRUID": self.recommendDruidConfigurations + } + parentRecommendConfDict.update(childRecommendConfDict) + return parentRecommendConfDict + + def recommendDruidConfigurations(self, configurations, clusterData, services, hosts): + + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist] + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + putCommonProperty = self.putProperty(configurations, "druid-common", services) + + putCommonProperty('druid.zk.service.host', self.getZKHostPortString(services)) + self.recommendDruidMaxMemoryLimitConfigurations(configurations, clusterData, services, hosts) + + # recommending the metadata storage uri + database_name = services['configurations']["druid-common"]["properties"]["database_name"] + metastore_hostname = services['configurations']["druid-common"]["properties"]["metastore_hostname"] + database_type = services['configurations']["druid-common"]["properties"]["druid.metadata.storage.type"] + metadata_storage_port = "1527" + mysql_extension_name = "io.druid.extensions:mysql-metadata-storage" + mysql_module_name = "mysql-metadata-storage" + postgres_module_name = "postgresql-metadata-storage" + extensions_load_list = services['configurations']['druid-common']['properties']['druid.extensions.loadList'] + extensions_pull_list = services['configurations']['druid-common']['properties']['druid.extensions.pullList'] + putDruidCommonProperty = self.putProperty(configurations, "druid-common", services) + + extensions_pull_list = self.removeFromList(extensions_pull_list, mysql_extension_name) + extensions_load_list = self.removeFromList(extensions_load_list, mysql_module_name) + extensions_load_list = self.removeFromList(extensions_load_list, postgres_module_name) + + if database_type == 'mysql': + metadata_storage_port = "3306" + extensions_pull_list = self.addToList(extensions_pull_list, mysql_extension_name) + extensions_load_list = self.addToList(extensions_load_list, mysql_module_name) + + if database_type == 'postgres': + extensions_load_list = self.addToList(extensions_load_list, postgres_module_name) + metadata_storage_port = "5432" + + putDruidCommonProperty('druid.metadata.storage.connector.port', metadata_storage_port) + putDruidCommonProperty('druid.metadata.storage.connector.connectURI', + self.getMetadataConnectionString(database_type).format(metastore_hostname, database_name, + metadata_storage_port)) + # HDFS is installed + if "HDFS" in servicesList and "hdfs-site" in services["configurations"]: + # recommend HDFS as default deep storage + extensions_load_list = self.addToList(extensions_load_list, "druid-hdfs-storage") + putCommonProperty("druid.storage.type", "hdfs") + putCommonProperty("druid.storage.storageDirectory", "/user/druid/data") + # configure indexer logs configs + putCommonProperty("druid.indexer.logs.type", "hdfs") + putCommonProperty("druid.indexer.logs.directory", "/user/druid/logs") + + if "KAFKA" in servicesList: + extensions_load_list = self.addToList(extensions_load_list, "druid-kafka-indexing-service") + + putCommonProperty('druid.extensions.loadList', extensions_load_list) + putCommonProperty('druid.extensions.pullList', extensions_pull_list) + + # JVM Configs go to env properties + putEnvProperty = self.putProperty(configurations, "druid-env", services) + + # processing thread pool Config + for component in ['DRUID_HISTORICAL', 'DRUID_BROKER']: + component_hosts = self.getHostsWithComponent("DRUID", component, services, hosts) + nodeType = self.DRUID_COMPONENT_NODE_TYPE_MAP[component] + putComponentProperty = self.putProperty(configurations, format("druid-{nodeType}"), services) + if (component_hosts is not None and len(component_hosts) > 0): + totalAvailableCpu = self.getMinCpu(component_hosts) + processingThreads = 1 + if totalAvailableCpu > 1: + processingThreads = totalAvailableCpu - 1 + putComponentProperty('druid.processing.numThreads', processingThreads) + putComponentProperty('druid.server.http.numThreads', max(10, (totalAvailableCpu * 17) / 16 + 2) + 30) + + def getMetadataConnectionString(self, database_type): + driverDict = { + 'mysql': 'jdbc:mysql://{0}:{2}/{1}?createDatabaseIfNotExist=true', + 'derby': 'jdbc:derby://{0}:{2}/{1};create=true', + 'postgres': 'jdbc:postgresql://{0}:{2}/{1}' + } + return driverDict.get(database_type.lower()) + + def addToList(self, json_list, word): + desr_list = json.loads(json_list) + if word not in desr_list: + desr_list.append(word) + return json.dumps(desr_list) + + def removeFromList(self, json_list, word): + desr_list = json.loads(json_list) + if word in desr_list: + desr_list.remove(word) + return json.dumps(desr_list) + + def recommendDruidMaxMemoryLimitConfigurations(self, configurations, clusterData, services, hosts): + putEnvPropertyAttribute = self.putPropertyAttribute(configurations, "druid-env") + for component in ["DRUID_HISTORICAL", "DRUID_MIDDLEMANAGER", "DRUID_BROKER", "DRUID_OVERLORD", + "DRUID_COORDINATOR"]: + component_hosts = self.getHostsWithComponent("DRUID", component, services, hosts) + if component_hosts is not None and len(component_hosts) > 0: + totalAvailableMem = self.getMinMemory(component_hosts) / 1024 # In MB + nodeType = self.DRUID_COMPONENT_NODE_TYPE_MAP[component] + putEnvPropertyAttribute(format('druid.{nodeType}.jvm.heap.memory'), 'maximum', + max(totalAvailableMem, 1024)) + + DRUID_COMPONENT_NODE_TYPE_MAP = { + 'DRUID_BROKER': 'broker', + 'DRUID_COORDINATOR': 'coordinator', + 'DRUID_HISTORICAL': 'historical', + 'DRUID_MIDDLEMANAGER': 'middlemanager', + 'DRUID_OVERLORD': 'overlord', + 'DRUID_ROUTER': 'router' + } + + def getMinMemory(self, component_hosts): + min_ram_kb = 1073741824 # 1 TB + for host in component_hosts: + ram_kb = host['Hosts']['total_mem'] + min_ram_kb = min(min_ram_kb, ram_kb) + return min_ram_kb + + def getMinCpu(self, component_hosts): + min_cpu = 256 + for host in component_hosts: + cpu_count = host['Hosts']['cpu_count'] + min_cpu = min(min_cpu, cpu_count) + return min_cpu + + def getServiceConfigurationValidators(self): + parentValidators = super(HDP26StackAdvisor, self).getServiceConfigurationValidators() + childValidators = { + "DRUID": {"druid-env": self.validateDruidEnvConfigurations, + "druid-historical": self.validateDruidHistoricalConfigurations, + "druid-broker": self.validateDruidBrokerConfigurations} + } + self.mergeValidators(parentValidators, childValidators) + return parentValidators + + def validateDruidEnvConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): + validationItems = [] + # Minimum Direct memory Validation + envProperties = services['configurations']['druid-env']['properties'] + for nodeType in ['broker', 'historical']: + properties = services['configurations'][format('druid-{nodeType}')]['properties'] + intermediateBufferSize = int(properties['druid.processing.buffer.sizeBytes']) / (1024 * 1024) # In MBs + processingThreads = int(properties['druid.processing.numThreads']) + directMemory = int(envProperties[format('druid.{nodeType}.jvm.direct.memory')]) + if directMemory < (processingThreads + 1) * intermediateBufferSize: + validationItems.extend( + {"config-name": format("druid.{nodeType}.jvm.direct.memory"), "item": self.getErrorItem( + format( + "Not enough direct memory available for {nodeType} Node." + "Please adjust druid.{nodeType}.jvm.direct.memory, druid.processing.buffer.sizeBytes, druid.processing.numThreads" + ) + ) + }) + return self.toConfigurationValidationProblems(validationItems, "druid-env") + + def validateDruidHistoricalConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): + validationItems = [ + {"config-name": "druid.processing.numThreads", + "item": self.validatorEqualsToRecommendedItem(properties, recommendedDefaults, + "druid.processing.numThreads")} + ] + return self.toConfigurationValidationProblems(validationItems, "druid-historical") + + def validateDruidBrokerConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): + validationItems = [ + {"config-name": "druid.processing.numThreads", + "item": self.validatorEqualsToRecommendedItem(properties, recommendedDefaults, + "druid.processing.numThreads")} + ] + return self.toConfigurationValidationProblems(validationItems, "druid-broker") http://git-wip-us.apache.org/repos/asf/ambari/blob/11cd9237/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py b/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py new file mode 100644 index 0000000..2988169 --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py @@ -0,0 +1,647 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import os + +from stacks.utils.RMFTestCase import * + +from mock.mock import MagicMock, patch +from resource_management.libraries import functions +from resource_management.libraries.functions import format +from resource_management.core.logger import Logger + +@patch("resource_management.libraries.Script.get_tmp_dir", new=MagicMock(return_value=('/var/lib/ambari-agent/tmp'))) [email protected](functions, "get_stack_version", new=MagicMock(return_value="2.0.0.0-1234")) +class TestDruid(RMFTestCase): + COMMON_SERVICES_PACKAGE_DIR = "DRUID/0.9.2/package" + STACK_VERSION = "2.6" + DEFAULT_IMMUTABLE_PATHS = ['/apps/hive/warehouse', '/apps/falcon', '/mr-history/done', '/app-logs', '/tmp'] + + def setUp(self): + Logger.logger = MagicMock() + self.testDirectory = os.path.dirname(os.path.abspath(__file__)) + self.num_times_to_iterate = 3 + self.wait_time = 1 + + def test_configure_overlord(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/overlord.py", + classname="DruidOverlord", + command="configure", + config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", + config_overrides = { 'role' : 'DRUID_OVERLORD' }, + stack_version=self.STACK_VERSION, + target=RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assert_configure_default('druid-overlord') + self.assertNoMoreResources() + + def test_start_overlord(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/overlord.py", + classname="DruidOverlord", + command="start", + config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", + stack_version=self.STACK_VERSION, + config_overrides = { 'role' : 'DRUID_OVERLORD' }, + target=RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assert_configure_default('druid-overlord') + self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-overlord/conf/druid-env.sh ; /usr/hdp/current/druid-overlord/bin/node.sh overlord start'), + user='druid' + ) + self.assertNoMoreResources() + + def test_stop_overlord(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/overlord.py", + classname="DruidOverlord", + command="stop", + config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", + stack_version=self.STACK_VERSION, + config_overrides = { 'role' : 'DRUID_OVERLORD' }, + target=RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-overlord/conf/druid-env.sh ; /usr/hdp/current/druid-overlord/bin/node.sh overlord stop'), + user='druid' + ) + self.assertNoMoreResources() + + def test_configure_coordinator(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/coordinator.py", + classname="DruidCoordinator", + command="configure", + config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", + config_overrides = { 'role' : 'DRUID_COORDINATOR' }, + stack_version=self.STACK_VERSION, + target=RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assert_configure_default('druid-coordinator') + self.assertNoMoreResources() + + def test_start_coordinator(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/coordinator.py", + classname="DruidCoordinator", + command="start", + config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", + stack_version=self.STACK_VERSION, + config_overrides = { 'role' : 'DRUID_COORDINATOR' }, + target=RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assert_configure_default('druid-coordinator') + self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-coordinator/conf/druid-env.sh ; /usr/hdp/current/druid-coordinator/bin/node.sh coordinator start'), + user='druid' + ) + self.assertNoMoreResources() + + def test_stop_coordinator(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/coordinator.py", + classname="DruidCoordinator", + command="stop", + config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", + stack_version=self.STACK_VERSION, + config_overrides = { 'role' : 'DRUID_COORDINATOR' }, + target=RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-coordinator/conf/druid-env.sh ; /usr/hdp/current/druid-coordinator/bin/node.sh coordinator stop'), + user='druid' + ) + self.assertNoMoreResources() + + def test_configure_broker(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/broker.py", + classname="DruidBroker", + command="configure", + config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", + config_overrides = { 'role' : 'DRUID_BROKER' }, + stack_version=self.STACK_VERSION, + target=RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assert_configure_default('druid-broker') + self.assertNoMoreResources() + + def test_start_broker(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/broker.py", + classname="DruidBroker", + command="start", + config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", + stack_version=self.STACK_VERSION, + config_overrides = { 'role' : 'DRUID_BROKER' }, + target=RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assert_configure_default('druid-broker') + self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-broker/conf/druid-env.sh ; /usr/hdp/current/druid-broker/bin/node.sh broker start'), + user='druid' + ) + self.assertNoMoreResources() + + def test_stop_broker(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/broker.py", + classname="DruidBroker", + command="stop", + config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", + stack_version=self.STACK_VERSION, + config_overrides = { 'role' : 'DRUID_BROKER' }, + target=RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-broker/conf/druid-env.sh ; /usr/hdp/current/druid-broker/bin/node.sh broker stop'), + user='druid' + ) + self.assertNoMoreResources() + + def test_configure_router(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/router.py", + classname="DruidRouter", + command="configure", + config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", + config_overrides = { 'role' : 'DRUID_ROUTER' }, + stack_version=self.STACK_VERSION, + target=RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assert_configure_default('druid-router') + self.assertNoMoreResources() + + def test_start_router(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/router.py", + classname="DruidRouter", + command="start", + config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", + stack_version=self.STACK_VERSION, + config_overrides = { 'role' : 'DRUID_ROUTER' }, + target=RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assert_configure_default('druid-router') + self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-router/conf/druid-env.sh ; /usr/hdp/current/druid-router/bin/node.sh router start'), + user='druid' + ) + self.assertNoMoreResources() + + def test_stop_router(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/router.py", + classname="DruidRouter", + command="stop", + config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", + stack_version=self.STACK_VERSION, + config_overrides = { 'role' : 'DRUID_ROUTER' }, + target=RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-router/conf/druid-env.sh ; /usr/hdp/current/druid-router/bin/node.sh router stop'), + user='druid' + ) + self.assertNoMoreResources() + + def test_configure_historical(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/historical.py", + classname="DruidHistorical", + command="configure", + config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", + config_overrides = { 'role' : 'DRUID_HISTORICAL' }, + stack_version=self.STACK_VERSION, + target=RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assert_configure_default('druid-historical') + self.assertNoMoreResources() + + def test_start_historical(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/historical.py", + classname="DruidHistorical", + command="start", + config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", + stack_version=self.STACK_VERSION, + config_overrides = { 'role' : 'DRUID_HISTORICAL' }, + target=RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assert_configure_default('druid-historical') + self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-historical/conf/druid-env.sh ; /usr/hdp/current/druid-historical/bin/node.sh historical start'), + user='druid' + ) + self.assertNoMoreResources() + + def test_stop_historical(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/historical.py", + classname="DruidHistorical", + command="stop", + config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", + stack_version=self.STACK_VERSION, + config_overrides = { 'role' : 'DRUID_HISTORICAL' }, + target=RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-historical/conf/druid-env.sh ; /usr/hdp/current/druid-historical/bin/node.sh historical stop'), + user='druid' + ) + self.assertNoMoreResources() + + def test_configure_middleManager(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/middlemanager.py", + classname="DruidMiddleManager", + command="configure", + config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", + config_overrides = { 'role' : 'DRUID_MIDDLEMANAGER' }, + stack_version=self.STACK_VERSION, + target=RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assert_configure_default('druid-middlemanager') + self.assertNoMoreResources() + + def test_start_middleManager(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/middlemanager.py", + classname="DruidMiddleManager", + command="start", + config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", + stack_version=self.STACK_VERSION, + config_overrides = { 'role' : 'DRUID_MIDDLEMANAGER' }, + target=RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assert_configure_default('druid-middlemanager') + self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-middlemanager/conf/druid-env.sh ; /usr/hdp/current/druid-middlemanager/bin/node.sh middleManager start'), + user='druid' + ) + self.assertNoMoreResources() + + def test_stop_middleManager(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/middlemanager.py", + classname="DruidMiddleManager", + command="stop", + config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json", + stack_version=self.STACK_VERSION, + config_overrides = { 'role' : 'DRUID_MIDDLEMANAGER' }, + target=RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-middlemanager/conf/druid-env.sh ; /usr/hdp/current/druid-middlemanager/bin/node.sh middleManager stop'), + user='druid' + ) + self.assertNoMoreResources() + + def assert_configure_default(self, role): + + self.assertResourceCalled('Directory', '/var/log/druid', + mode=0755, + cd_access='a', + owner='druid', + group='hadoop', + create_parents=True, + recursive_ownership=True + ) + + self.assertResourceCalled('Directory', '/var/run/druid', + mode=0755, + cd_access='a', + owner='druid', + group='hadoop', + create_parents=True, + recursive_ownership=True + ) + + self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/_common'), + mode=0755, + cd_access='a', + owner='druid', + group='hadoop', + create_parents=True, + recursive_ownership=True + ) + + self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/coordinator'), + mode=0755, + cd_access='a', + owner='druid', + group='hadoop', + create_parents=True, + recursive_ownership=True + ) + + self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/broker'), + mode=0755, + cd_access='a', + owner='druid', + group='hadoop', + create_parents=True, + recursive_ownership=True + ) + + self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/middleManager'), + mode=0755, + cd_access='a', + owner='druid', + group='hadoop', + create_parents=True, + recursive_ownership=True + ) + + self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/historical'), + mode=0755, + cd_access='a', + owner='druid', + group='hadoop', + create_parents=True, + recursive_ownership=True + ) + + self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/overlord'), + mode=0755, + cd_access='a', + owner='druid', + group='hadoop', + create_parents=True, + recursive_ownership=True + ) + + self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/router'), + mode=0755, + cd_access='a', + owner='druid', + group='hadoop', + create_parents=True, + recursive_ownership=True + ) + + self.assertResourceCalled('Directory', '/apps/druid/segmentCache', + mode=0755, + cd_access='a', + owner='druid', + group='hadoop', + create_parents=True, + recursive_ownership=True + ) + + self.assertResourceCalled('File', format('/usr/hdp/current/{role}/conf/druid-env.sh'), + owner = 'druid', + content = InlineTemplate(self.getConfig()['configurations']['druid-env']['content']) + ) + druid_common_config = mutable_config_dict(self.getConfig()['configurations']['druid-common']) + druid_common_config['druid.host'] = 'c6401.ambari.apache.org' + druid_common_config['druid.extensions.directory'] = format('/usr/hdp/current/{role}/extensions') + druid_common_config['druid.extensions.hadoopDependenciesDir'] = format('/usr/hdp/current/{role}/hadoop-dependencies') + druid_common_config['druid.selectors.indexing.serviceName'] = 'druid/overlord' + druid_common_config['druid.selectors.coordinator.serviceName'] = 'druid/coordinator' + + self.assertResourceCalled('PropertiesFile', 'common.runtime.properties', + dir=format("/usr/hdp/current/{role}/conf/_common"), + properties=druid_common_config, + owner='druid', + group='hadoop' + ) + + self.assertResourceCalled('File', format('/usr/hdp/current/{role}/conf/_common/druid-log4j.xml'), + mode=0644, + owner = 'druid', + group = 'hadoop', + content = InlineTemplate(self.getConfig()['configurations']['druid-log4j']['content']) + ) + + self.assertResourceCalled('File', '/etc/logrotate.d/druid', + mode=0644, + owner = 'root', + group = 'root', + content = InlineTemplate(self.getConfig()['configurations']['druid-logrotate']['content']) + ) + + self.assertResourceCalled('XmlConfig', "core-site.xml", + conf_dir=format('/usr/hdp/current/{role}/conf/_common'), + configurations=self.getConfig()['configurations']['core-site'], + configuration_attributes=self.getConfig()['configuration_attributes']['core-site'], + owner='druid', + group='hadoop' + ) + + self.assertResourceCalled('XmlConfig', "yarn-site.xml", + conf_dir=format('/usr/hdp/current/{role}/conf/_common'), + configurations=self.getConfig()['configurations']['yarn-site'], + configuration_attributes=self.getConfig()['configuration_attributes']['yarn-site'], + owner='druid', + group='hadoop' + ) + + self.assertResourceCalled('XmlConfig', "hdfs-site.xml", + conf_dir=format('/usr/hdp/current/{role}/conf/_common'), + configurations=self.getConfig()['configurations']['hdfs-site'], + configuration_attributes=self.getConfig()['configuration_attributes']['hdfs-site'], + owner='druid', + group='hadoop' + ) + + self.assertResourceCalled('PropertiesFile', "runtime.properties", + dir=format('/usr/hdp/current/{role}/conf/coordinator'), + properties=self.getConfig()['configurations']['druid-coordinator'], + owner='druid', + group='hadoop' + ) + + self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/coordinator/jvm.config"), + owner='druid', + group='hadoop', + content=InlineTemplate("-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}", + node_heap_memory=1024, + node_direct_memory=2048, + node_jvm_opts='-Duser.timezone=UTC -Dfile.encoding=UTF-8', + log4j_config_file=format('/usr/hdp/current/{role}/conf/_common/druid-log4j.xml') + ) + ) + + self.assertResourceCalled('PropertiesFile', "runtime.properties", + dir=format('/usr/hdp/current/{role}/conf/overlord'), + properties=self.getConfig()['configurations']['druid-overlord'], + owner='druid', + group='hadoop' + ) + + self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/overlord/jvm.config"), + owner='druid', + group='hadoop', + content=InlineTemplate("-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}", + node_heap_memory=1024, + node_direct_memory=2048, + node_jvm_opts='-Duser.timezone=UTC -Dfile.encoding=UTF-8', + log4j_config_file=format('/usr/hdp/current/{role}/conf/_common/druid-log4j.xml') + ) + ) + + self.assertResourceCalled('PropertiesFile', "runtime.properties", + dir=format('/usr/hdp/current/{role}/conf/historical'), + properties=self.getConfig()['configurations']['druid-historical'], + owner='druid', + group='hadoop' + ) + + self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/historical/jvm.config"), + owner='druid', + group='hadoop', + content=InlineTemplate("-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}", + node_heap_memory=1024, + node_direct_memory=2048, + node_jvm_opts='-Duser.timezone=UTC -Dfile.encoding=UTF-8', + log4j_config_file=format('/usr/hdp/current/{role}/conf/_common/druid-log4j.xml') + ) + ) + + + self.assertResourceCalled('PropertiesFile', "runtime.properties", + dir=format('/usr/hdp/current/{role}/conf/broker'), + properties=self.getConfig()['configurations']['druid-broker'], + owner='druid', + group='hadoop' + ) + + self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/broker/jvm.config"), + owner='druid', + group='hadoop', + content=InlineTemplate("-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}", + node_heap_memory=1024, + node_direct_memory=2048, + node_jvm_opts='-Duser.timezone=UTC -Dfile.encoding=UTF-8', + log4j_config_file=format('/usr/hdp/current/{role}/conf/_common/druid-log4j.xml') + ) + ) + + + self.assertResourceCalled('PropertiesFile', "runtime.properties", + dir=format('/usr/hdp/current/{role}/conf/middleManager'), + properties=self.getConfig()['configurations']['druid-middlemanager'], + owner='druid', + group='hadoop' + ) + + self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/middleManager/jvm.config"), + owner='druid', + group='hadoop', + content=InlineTemplate("-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}", + node_heap_memory=1024, + node_direct_memory=2048, + node_jvm_opts='-Duser.timezone=UTC -Dfile.encoding=UTF-8', + log4j_config_file=format('/usr/hdp/current/{role}/conf/_common/druid-log4j.xml') + ) + ) + + self.assertResourceCalled('PropertiesFile', "runtime.properties", + dir=format('/usr/hdp/current/{role}/conf/router'), + properties=self.getConfig()['configurations']['druid-router'], + owner='druid', + group='hadoop' + ) + + self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/router/jvm.config"), + owner='druid', + group='hadoop', + content=InlineTemplate("-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}", + node_heap_memory=1024, + node_direct_memory=2048, + node_jvm_opts='-Duser.timezone=UTC -Dfile.encoding=UTF-8', + log4j_config_file=format('/usr/hdp/current/{role}/conf/_common/druid-log4j.xml') + ) + ) + + self.assertResourceCalled('HdfsResource', '/user/druid', + immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, + security_enabled = False, + hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', + keytab = UnknownConfigurationMock(), + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + hdfs_site = {u'a': u'b'}, + kinit_path_local = '/usr/bin/kinit', + principal_name = 'missing_principal', + user = 'hdfs', + owner = 'druid', + hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', + type = 'directory', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', + dfs_type = '', + recursive_chown=True, + recursive_chmod=True + ) + + self.assertResourceCalled('HdfsResource', '/user/druid/data', + immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, + security_enabled = False, + hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', + keytab = UnknownConfigurationMock(), + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + hdfs_site = {u'a': u'b'}, + kinit_path_local = '/usr/bin/kinit', + principal_name = 'missing_principal', + user = 'hdfs', + owner = 'druid', + hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', + type = 'directory', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', + dfs_type = '', + mode=0755 + ) + + self.assertResourceCalled('HdfsResource', '/tmp/druid-indexing', + immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, + security_enabled = False, + hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', + keytab = UnknownConfigurationMock(), + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + hdfs_site = {u'a': u'b'}, + kinit_path_local = '/usr/bin/kinit', + principal_name = 'missing_principal', + user = 'hdfs', + owner = 'druid', + hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', + type = 'directory', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', + dfs_type = '', + mode=0755 + ) + + self.assertResourceCalled('HdfsResource', '/user/druid/logs', + immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, + security_enabled = False, + hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', + keytab = UnknownConfigurationMock(), + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + hdfs_site = {u'a': u'b'}, + kinit_path_local = '/usr/bin/kinit', + principal_name = 'missing_principal', + user = 'hdfs', + owner = 'druid', + hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', + type = 'directory', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', + dfs_type = '', + mode=0755 + ) + + self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/extensions'), + mode=0755, + cd_access='a', + owner='druid', + group='hadoop', + create_parents=True, + recursive_ownership=True + ) + + self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/hadoop-dependencies'), + mode=0755, + cd_access='a', + owner='druid', + group='hadoop', + create_parents=True, + recursive_ownership=True + ) + + self.assertResourceCalled('Execute', format("source /usr/hdp/current/{role}/conf/druid-env.sh ; java -classpath '/usr/hdp/current/{role}/lib/*' -Ddruid.extensions.loadList=[] -Ddruid.extensions.directory=/usr/hdp/current/{role}/extensions -Ddruid.extensions.hadoopDependenciesDir=/usr/hdp/current/{role}/hadoop-dependencies io.druid.cli.Main tools pull-deps -c mysql-metadata-storage --no-default-hadoop"), + user='druid' + ) + + +def mutable_config_dict(config): + rv = {} + for key, value in config.iteritems(): + rv[key] = value + return rv http://git-wip-us.apache.org/repos/asf/ambari/blob/11cd9237/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py new file mode 100644 index 0000000..8227d69 --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py @@ -0,0 +1,575 @@ +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import json +import os +from unittest import TestCase +from mock.mock import patch + + +class TestHDP26StackAdvisor(TestCase): + def setUp(self): + import imp + self.maxDiff = None + self.testDirectory = os.path.dirname(os.path.abspath(__file__)) + stackAdvisorPath = os.path.join(self.testDirectory, '../../../../../main/resources/stacks/stack_advisor.py') + hdp206StackAdvisorPath = os.path.join(self.testDirectory, + '../../../../../main/resources/stacks/HDP/2.0.6/services/stack_advisor.py') + hdp21StackAdvisorPath = os.path.join(self.testDirectory, + '../../../../../main/resources/stacks/HDP/2.1/services/stack_advisor.py') + hdp22StackAdvisorPath = os.path.join(self.testDirectory, + '../../../../../main/resources/stacks/HDP/2.2/services/stack_advisor.py') + hdp23StackAdvisorPath = os.path.join(self.testDirectory, + '../../../../../main/resources/stacks/HDP/2.3/services/stack_advisor.py') + hdp24StackAdvisorPath = os.path.join(self.testDirectory, + '../../../../../main/resources/stacks/HDP/2.4/services/stack_advisor.py') + hdp25StackAdvisorPath = os.path.join(self.testDirectory, + '../../../../../main/resources/stacks/HDP/2.5/services/stack_advisor.py') + hdp26StackAdvisorPath = os.path.join(self.testDirectory, + '../../../../../main/resources/stacks/HDP/2.6/services/stack_advisor.py') + hdp26StackAdvisorClassName = 'HDP26StackAdvisor' + + with open(stackAdvisorPath, 'rb') as fp: + imp.load_module('stack_advisor', fp, stackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) + with open(hdp206StackAdvisorPath, 'rb') as fp: + imp.load_module('stack_advisor_impl', fp, hdp206StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) + with open(hdp21StackAdvisorPath, 'rb') as fp: + imp.load_module('stack_advisor_impl', fp, hdp21StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) + with open(hdp22StackAdvisorPath, 'rb') as fp: + imp.load_module('stack_advisor_impl', fp, hdp22StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) + with open(hdp23StackAdvisorPath, 'rb') as fp: + imp.load_module('stack_advisor_impl', fp, hdp23StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) + with open(hdp24StackAdvisorPath, 'rb') as fp: + imp.load_module('stack_advisor_impl', fp, hdp24StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) + with open(hdp25StackAdvisorPath, 'rb') as fp: + imp.load_module('stack_advisor_impl', fp, hdp25StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) + with open(hdp26StackAdvisorPath, 'rb') as fp: + stack_advisor_impl = imp.load_module('stack_advisor_impl', fp, hdp26StackAdvisorPath, + ('.py', 'rb', imp.PY_SOURCE)) + clazz = getattr(stack_advisor_impl, hdp26StackAdvisorClassName) + self.stackAdvisor = clazz() + + # substitute method in the instance + self.get_system_min_uid_real = self.stackAdvisor.get_system_min_uid + self.stackAdvisor.get_system_min_uid = self.get_system_min_uid_magic + + @patch('__builtin__.open') + @patch('os.path.exists') + def get_system_min_uid_magic(self, exists_mock, open_mock): + class MagicFile(object): + def read(self): + return """ + #test line UID_MIN 200 + UID_MIN 500 + """ + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + + def __enter__(self): + return self + + exists_mock.return_value = True + open_mock.return_value = MagicFile() + return self.get_system_min_uid_real() + + def test_recommendDruidConfigurations_withMysql(self): + hosts = { + "items": [ + { + "Hosts": { + "cpu_count": 4, + "total_mem": 50331648, + "disk_info": [ + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"}, + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"} + ], + "public_host_name": "c6401.ambari.apache.org", + "host_name": "c6401.ambari.apache.org" + }, + } + ] + } + + services = { + "Versions": { + "parent_stack_version": "2.5", + "stack_name": "HDP", + "stack_version": "2.6", + "stack_hierarchy": { + "stack_name": "HDP", + "stack_versions": ["2.5", "2.4", "2.3", "2.2", "2.1", "2.0.6"] + } + }, + "services": [{ + "StackServices": { + "service_name": "DRUID", + }, + "components": [ + { + "StackServiceComponents": { + "component_name": "DRUID_COORDINATOR", + "hostnames": ["c6401.ambari.apache.org"] + }, + }, + { + "StackServiceComponents": { + "component_name": "DRUID_OVERLORD", + "hostnames": ["c6401.ambari.apache.org"] + }, + }, + { + "StackServiceComponents": { + "component_name": "DRUID_BROKER", + "hostnames": ["c6401.ambari.apache.org"] + }, + }, + { + "StackServiceComponents": { + "component_name": "DRUID_HISTORICAL", + "hostnames": ["c6401.ambari.apache.org"] + }, + }, + { + "StackServiceComponents": { + "component_name": "DRUID_MIDDLEMANAGER", + "hostnames": ["c6401.ambari.apache.org"] + }, + } + ] + } + ], + "configurations": { + "druid-common": { + "properties": { + "database_name": "druid", + "metastore_hostname": "c6401.ambari.apache.org", + "druid.metadata.storage.type": "mysql", + "druid.extensions.loadList": "[\"postgresql-metadata-storage\"]", + "druid.extensions.pullList": "[]" + } + } + } + } + + clusterData = { + "cpu": 4, + "mapMemory": 30000, + "amMemory": 20000, + "reduceMemory": 20560, + "containers": 30, + "ramPerContainer": 512, + "referenceNodeManagerHost": { + "total_mem": 10240 * 1024 + } + } + + configurations = { + } + + self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations, + {'druid-historical': { + 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}}, + 'druid-broker': { + 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}}, + 'druid-common': {'properties': {'druid.extensions.loadList': '["mysql-metadata-storage"]', + 'druid.metadata.storage.connector.port': '3306', + 'druid.metadata.storage.connector.connectURI': 'jdbc:mysql://c6401.ambari.apache.org:3306/druid?createDatabaseIfNotExist=true', + 'druid.zk.service.host': '', + 'druid.extensions.pullList': '["io.druid.extensions:mysql-metadata-storage"]'}}, + 'druid-env': {'properties': {}, + 'property_attributes': {'druid.coordinator.jvm.heap.memory': {'maximum': '49152'}, + 'druid.overlord.jvm.heap.memory': {'maximum': '49152'}, + 'druid.middlemanager.jvm.heap.memory': { + 'maximum': '49152'}, + 'druid.historical.jvm.heap.memory': {'maximum': '49152'}, + 'druid.broker.jvm.heap.memory': {'maximum': '49152'}}}} + ) + + def test_recommendDruidConfigurations_WithPostgresql(self): + hosts = { + "items": [ + { + "Hosts": { + "cpu_count": 4, + "total_mem": 50331648, + "disk_info": [ + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"}, + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"} + ], + "public_host_name": "c6401.ambari.apache.org", + "host_name": "c6401.ambari.apache.org" + } + } + ] + } + + services = { + "Versions": { + "parent_stack_version": "2.5", + "stack_name": "HDP", + "stack_version": "2.6", + "stack_hierarchy": { + "stack_name": "HDP", + "stack_versions": ["2.5", "2.4", "2.3", "2.2", "2.1", "2.0.6"] + } + }, + "services": [{ + "StackServices": { + "service_name": "DRUID", + }, + "components": [ + { + "StackServiceComponents": { + "component_name": "DRUID_COORDINATOR", + "hostnames": ["c6401.ambari.apache.org"] + }, + }, + { + "StackServiceComponents": { + "component_name": "DRUID_OVERLORD", + "hostnames": ["c6401.ambari.apache.org"] + }, + }, + { + "StackServiceComponents": { + "component_name": "DRUID_BROKER", + "hostnames": ["c6401.ambari.apache.org"] + }, + }, + { + "StackServiceComponents": { + "component_name": "DRUID_HISTORICAL", + "hostnames": ["c6401.ambari.apache.org"] + }, + }, + { + "StackServiceComponents": { + "component_name": "DRUID_MIDDLEMANAGER", + "hostnames": ["c6401.ambari.apache.org"] + }, + } + ] + } + ], + "configurations": { + "druid-common": { + "properties": { + "database_name": "druid", + "metastore_hostname": "c6401.ambari.apache.org", + "druid.metadata.storage.type": "postgres", + "druid.extensions.loadList": "[\"mysql-metadata-storage\"]", + "druid.extensions.pullList": "[]" + } + } + } + } + + clusterData = { + } + + configurations = { + } + + self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations, + {'druid-historical': { + 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}}, + 'druid-broker': { + 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}}, + 'druid-common': {'properties': {'druid.extensions.loadList': '["postgresql-metadata-storage"]', + 'druid.metadata.storage.connector.port': '5432', + 'druid.metadata.storage.connector.connectURI': 'jdbc:postgresql://c6401.ambari.apache.org:5432/druid', + 'druid.zk.service.host': '', + 'druid.extensions.pullList': '[]'}}, + 'druid-env': {'properties': {}, + 'property_attributes': {'druid.coordinator.jvm.heap.memory': {'maximum': '49152'}, + 'druid.overlord.jvm.heap.memory': {'maximum': '49152'}, + 'druid.middlemanager.jvm.heap.memory': { + 'maximum': '49152'}, + 'druid.historical.jvm.heap.memory': {'maximum': '49152'}, + 'druid.broker.jvm.heap.memory': {'maximum': '49152'}}}} + ) + + def test_recommendDruidConfigurations_WithDerby(self): + hosts = { + "items": [ + { + "Hosts": { + "cpu_count": 4, + "total_mem": 50331648, + "disk_info": [ + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"}, + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"} + ], + "public_host_name": "c6401.ambari.apache.org", + "host_name": "c6401.ambari.apache.org" + } + } + ] + } + + services = { + "Versions": { + "parent_stack_version": "2.5", + "stack_name": "HDP", + "stack_version": "2.6", + "stack_hierarchy": { + "stack_name": "HDP", + "stack_versions": ["2.5", "2.4", "2.3", "2.2", "2.1", "2.0.6"] + } + }, + "services": [{ + "StackServices": { + "service_name": "DRUID", + }, + "components": [ + { + "StackServiceComponents": { + "component_name": "DRUID_COORDINATOR", + "hostnames": ["c6401.ambari.apache.org"] + }, + }, + { + "StackServiceComponents": { + "component_name": "DRUID_OVERLORD", + "hostnames": ["c6401.ambari.apache.org"] + }, + }, + { + "StackServiceComponents": { + "component_name": "DRUID_BROKER", + "hostnames": ["c6401.ambari.apache.org"] + }, + }, + { + "StackServiceComponents": { + "component_name": "DRUID_HISTORICAL", + "hostnames": ["c6401.ambari.apache.org"] + }, + }, + { + "StackServiceComponents": { + "component_name": "DRUID_MIDDLEMANAGER", + "hostnames": ["c6401.ambari.apache.org"] + }, + } + ] + } + ], + "configurations": { + "druid-common": { + "properties": { + "database_name": "druid", + "metastore_hostname": "c6401.ambari.apache.org", + "druid.metadata.storage.type": "derby", + "druid.extensions.loadList": "[\"mysql-metadata-storage\"]", + "druid.extensions.pullList": "[]" + } + } + } + } + + clusterData = { + } + + configurations = { + } + + self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations, + {'druid-historical': { + 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}}, + 'druid-broker': { + 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}}, + 'druid-common': {'properties': {'druid.extensions.loadList': '[]', + 'druid.metadata.storage.connector.port': '1527', + 'druid.metadata.storage.connector.connectURI': 'jdbc:derby://c6401.ambari.apache.org:1527/druid;create=true', + 'druid.zk.service.host': '', + 'druid.extensions.pullList': '[]'}}, + 'druid-env': {'properties': {}, + 'property_attributes': {'druid.coordinator.jvm.heap.memory': {'maximum': '49152'}, + 'druid.overlord.jvm.heap.memory': {'maximum': '49152'}, + 'druid.middlemanager.jvm.heap.memory': { + 'maximum': '49152'}, + 'druid.historical.jvm.heap.memory': {'maximum': '49152'}, + 'druid.broker.jvm.heap.memory': {'maximum': '49152'}}}} + ) + + def test_recommendDruidConfigurations_heterogeneous_hosts(self): + hosts = { + "items": [ + { + "href": "/api/v1/hosts/c6401.ambari.apache.org", + "Hosts": { + "cpu_count": 4, + "total_mem": 50331648, + "disk_info": [ + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"}, + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"} + ], + "public_host_name": "c6401.ambari.apache.org", + "host_name": "c6401.ambari.apache.org" + } + }, { + "href": "/api/v1/hosts/c6402.ambari.apache.org", + "Hosts": { + "cpu_count": 1, + "total_mem": 1922680, + "disk_info": [ + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"}, + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"} + ], + "public_host_name": "c6402.ambari.apache.org", + "host_name": "c6402.ambari.apache.org" + } + }, + { + "href": "/api/v1/hosts/c6403.ambari.apache.org", + "Hosts": { + "cpu_count": 3, + "total_mem": 3845360, + "disk_info": [ + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"}, + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"} + ], + "public_host_name": "c6403.ambari.apache.org", + "host_name": "c6403.ambari.apache.org" + } + } + ] + } + + services = { + "Versions": { + "parent_stack_version": "2.5", + "stack_name": "HDP", + "stack_version": "2.6", + "stack_hierarchy": { + "stack_name": "HDP", + "stack_versions": ["2.5", "2.4", "2.3", "2.2", "2.1", "2.0.6"] + } + }, + "services": [{ + "StackServices": { + "service_name": "DRUID", + }, + "components": [ + { + "StackServiceComponents": { + "component_name": "DRUID_COORDINATOR", + "hostnames": ["c6401.ambari.apache.org"] + }, + }, + { + "StackServiceComponents": { + "component_name": "DRUID_OVERLORD", + "hostnames": ["c6401.ambari.apache.org"] + }, + }, + { + "StackServiceComponents": { + "component_name": "DRUID_BROKER", + "hostnames": ["c6402.ambari.apache.org", "c6403.ambari.apache.org"] + }, + }, + { + "StackServiceComponents": { + "component_name": "DRUID_HISTORICAL", + "hostnames": ["c6401.ambari.apache.org", "c6403.ambari.apache.org"] + }, + }, + { + "StackServiceComponents": { + "component_name": "DRUID_MIDDLEMANAGER", + "hostnames": ["c6401.ambari.apache.org"] + }, + } + ] + } + ], + "configurations": { + "druid-common": { + "properties": { + "database_name": "druid", + "metastore_hostname": "c6401.ambari.apache.org", + "druid.metadata.storage.type": "derby", + "druid.extensions.loadList": "[\"mysql-metadata-storage\"]", + "druid.extensions.pullList": "[]" + } + } + } + } + + clusterData = { + } + + configurations = { + } + + self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations, + {'druid-historical': { + 'properties': {'druid.processing.numThreads': '2', 'druid.server.http.numThreads': '40'}}, + 'druid-broker': { + 'properties': {'druid.processing.numThreads': '1', 'druid.server.http.numThreads': '40'}}, + 'druid-common': {'properties': {'druid.extensions.loadList': '[]', + 'druid.metadata.storage.connector.port': '1527', + 'druid.metadata.storage.connector.connectURI': 'jdbc:derby://c6401.ambari.apache.org:1527/druid;create=true', + 'druid.zk.service.host': '', + 'druid.extensions.pullList': '[]'}}, + 'druid-env': {'properties': {}, + 'property_attributes': {'druid.coordinator.jvm.heap.memory': {'maximum': '49152'}, + 'druid.overlord.jvm.heap.memory': {'maximum': '49152'}, + 'druid.middlemanager.jvm.heap.memory': { + 'maximum': '49152'}, + 'druid.historical.jvm.heap.memory': {'maximum': '3755'}, + 'druid.broker.jvm.heap.memory': {'maximum': '1877'}}}} + ) + + +def load_json(self, filename): + file = os.path.join(self.testDirectory, filename) + with open(file, 'rb') as f: + data = json.load(f) + return data
