AMBARI-22125. Better defaults for druid memory configs to avoid OutOfMemoryErrors on small machines
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1da77356 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1da77356 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1da77356 Branch: refs/heads/branch-feature-AMBARI-14714 Commit: 1da77356b0de98df51f48bedc087daf01aedbd7c Parents: d86f764 Author: Nishant <[email protected]> Authored: Thu Oct 5 01:21:00 2017 +0530 Committer: Nishant <[email protected]> Committed: Thu Oct 5 01:21:00 2017 +0530 ---------------------------------------------------------------------- .../DRUID/0.9.2/configuration/druid-broker.xml | 6 + .../0.9.2/configuration/druid-historical.xml | 6 + .../stacks/HDP/2.6/services/stack_advisor.py | 28 ++- .../stacks/2.6/common/test_stack_advisor.py | 190 ++++++++++++++++++- 4 files changed, 219 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/1da77356/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-broker.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-broker.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-broker.xml index 4f05da0..6146ca3 100644 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-broker.xml +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-broker.xml @@ -69,6 +69,12 @@ <on-ambari-upgrade add="false"/> </property> <property> + <name>druid.processing.numMergeBuffers</name> + <value>2</value> + <description>The number of direct memory buffers available for merging query results. The buffers are sized by druid.processing.buffer.sizeBytes.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> <name>druid.broker.cache.useCache</name> <value>true</value> <description>Enable the cache on the broker.</description> http://git-wip-us.apache.org/repos/asf/ambari/blob/1da77356/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml index 9b65404..5ff30ce 100644 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml @@ -39,6 +39,12 @@ <on-ambari-upgrade add="false"/> </property> <property> + <name>druid.processing.numMergeBuffers</name> + <value>2</value> + <description>The number of direct memory buffers available for merging query results. The buffers are sized by druid.processing.buffer.sizeBytes.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> <name>druid.processing.buffer.sizeBytes</name> <value>1073741824</value> <value-attributes> http://git-wip-us.apache.org/repos/asf/ambari/blob/1da77356/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py index 723ff4e..0d2925e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py @@ -16,9 +16,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import math - import json +import math import re from resource_management.libraries.functions import format @@ -144,7 +143,7 @@ class HDP26StackAdvisor(HDP25StackAdvisor): # JVM Configs go to env properties putEnvProperty = self.putProperty(configurations, "druid-env", services) - # processing thread pool Config + # processing thread pool and memory configs for component in ['DRUID_HISTORICAL', 'DRUID_BROKER']: component_hosts = self.getHostsWithComponent("DRUID", component, services, hosts) nodeType = self.DRUID_COMPONENT_NODE_TYPE_MAP[component] @@ -154,8 +153,31 @@ class HDP26StackAdvisor(HDP25StackAdvisor): processingThreads = 1 if totalAvailableCpu > 1: processingThreads = totalAvailableCpu - 1 + numMergeBuffers = max(2, processingThreads/4) putComponentProperty('druid.processing.numThreads', processingThreads) putComponentProperty('druid.server.http.numThreads', max(10, (totalAvailableCpu * 17) / 16 + 2) + 30) + putComponentProperty('druid.processing.numMergeBuffers', numMergeBuffers) + totalAvailableMemInMb = self.getMinMemory(component_hosts) / 1024 + maxAvailableBufferSizeInMb = totalAvailableMemInMb/(processingThreads + numMergeBuffers) + putComponentProperty('druid.processing.buffer.sizeBytes', self.getDruidProcessingBufferSizeInMb(totalAvailableMemInMb) * 1024 * 1024) + + + # returns the recommended druid processing buffer size in Mb. + # the recommended buffer size is kept lower then the max available memory to have enough free memory to load druid data. + # for low memory nodes, the actual allocated buffer size is small to keep some free memory for memory mapping of segments + # If user installs all druid processes on a single node, memory available for loading segments will be further decreased. + def getDruidProcessingBufferSizeInMb(self, maxAvailableBufferSizeInMb): + if maxAvailableBufferSizeInMb <= 256: + return min(64, maxAvailableBufferSizeInMb) + elif maxAvailableBufferSizeInMb <= 1024: + return 128 + elif maxAvailableBufferSizeInMb <= 2048: + return 256 + elif maxAvailableBufferSizeInMb <= 6144: + return 512 + # High Memory nodes below + else : + return 1024 def recommendSupersetConfigurations(self, configurations, clusterData, services, hosts): # superset is in list of services to be installed http://git-wip-us.apache.org/repos/asf/ambari/blob/1da77356/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py index f8483c1..ec44b3d 100644 --- a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py @@ -188,9 +188,15 @@ class TestHDP26StackAdvisor(TestCase): self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, {'druid-historical': { - 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}}, + 'properties': {'druid.processing.numThreads': '3', + 'druid.server.http.numThreads': '40', + 'druid.processing.numMergeBuffers': '2', + 'druid.processing.buffer.sizeBytes': '1073741824'}}, 'druid-broker': { - 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}}, + 'properties': {'druid.processing.numThreads': '3', + 'druid.server.http.numThreads': '40', + 'druid.processing.numMergeBuffers': '2', + 'druid.processing.buffer.sizeBytes': '1073741824'}}, 'druid-common': {'properties': {'druid.extensions.loadList': '["mysql-metadata-storage"]', 'druid.metadata.storage.connector.port': '3306', 'druid.metadata.storage.connector.connectURI': 'jdbc:mysql://c6401.ambari.apache.org:3306/druid?createDatabaseIfNotExist=true', @@ -767,9 +773,15 @@ class TestHDP26StackAdvisor(TestCase): self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, {'druid-historical': { - 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}}, + 'properties': {'druid.processing.numThreads': '3', + 'druid.server.http.numThreads': '40', + 'druid.processing.numMergeBuffers': '2', + 'druid.processing.buffer.sizeBytes': '1073741824'}}, 'druid-broker': { - 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}}, + 'properties': {'druid.processing.numThreads': '3', + 'druid.server.http.numThreads': '40', + 'druid.processing.numMergeBuffers': '2', + 'druid.processing.buffer.sizeBytes': '1073741824'}}, 'druid-common': {'properties': {'druid.extensions.loadList': '["postgresql-metadata-storage"]', 'druid.metadata.storage.connector.port': '5432', 'druid.metadata.storage.connector.connectURI': 'jdbc:postgresql://c6401.ambari.apache.org:5432/druid', @@ -875,9 +887,15 @@ class TestHDP26StackAdvisor(TestCase): self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, {'druid-historical': { - 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}}, + 'properties': {'druid.processing.numThreads': '3', + 'druid.server.http.numThreads': '40', + 'druid.processing.numMergeBuffers': '2', + 'druid.processing.buffer.sizeBytes': '1073741824'}}, 'druid-broker': { - 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}}, + 'properties': {'druid.processing.numThreads': '3', + 'druid.server.http.numThreads': '40', + 'druid.processing.numMergeBuffers': '2', + 'druid.processing.buffer.sizeBytes': '1073741824'}}, 'druid-common': {'properties': {'druid.extensions.loadList': '[]', 'druid.metadata.storage.connector.port': '1527', 'druid.metadata.storage.connector.connectURI': 'jdbc:derby://c6401.ambari.apache.org:1527/druid;create=true', @@ -892,6 +910,7 @@ class TestHDP26StackAdvisor(TestCase): ) + def test_recommendDruidConfigurations_property_existence_check(self): # Test for https://issues.apache.org/jira/browse/AMBARI-19144 hosts = { @@ -1069,9 +1088,15 @@ class TestHDP26StackAdvisor(TestCase): self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, {'druid-historical': { - 'properties': {'druid.processing.numThreads': '2', 'druid.server.http.numThreads': '40'}}, + 'properties': {'druid.processing.numThreads': '2', + 'druid.server.http.numThreads': '40', + 'druid.processing.numMergeBuffers': '2', + 'druid.processing.buffer.sizeBytes': '536870912'}}, 'druid-broker': { - 'properties': {'druid.processing.numThreads': '1', 'druid.server.http.numThreads': '40'}}, + 'properties': {'druid.processing.numThreads': '1', + 'druid.server.http.numThreads': '40', + 'druid.processing.numMergeBuffers': '2', + 'druid.processing.buffer.sizeBytes': '268435456'}}, 'druid-common': {'properties': {'druid.extensions.loadList': '[]', 'druid.metadata.storage.connector.port': '1527', 'druid.metadata.storage.connector.connectURI': 'jdbc:derby://c6401.ambari.apache.org:1527/druid;create=true', @@ -1086,6 +1111,155 @@ class TestHDP26StackAdvisor(TestCase): 'druid.broker.jvm.heap.memory': {'maximum': '1877'}}}} ) + def test_recommendDruidConfigurations_low_mem_hosts(self): + hosts = { + "items": [ + { + "href": "/api/v1/hosts/c6401.ambari.apache.org", + "Hosts": { + "cpu_count": 8, + "total_mem": 102400, + "disk_info": [ + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"}, + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"} + ], + "public_host_name": "c6401.ambari.apache.org", + "host_name": "c6401.ambari.apache.org" + } + }, { + "href": "/api/v1/hosts/c6402.ambari.apache.org", + "Hosts": { + "cpu_count": 4, + "total_mem": 204800, + "disk_info": [ + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"}, + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"} + ], + "public_host_name": "c6402.ambari.apache.org", + "host_name": "c6402.ambari.apache.org" + } + }, + { + "href": "/api/v1/hosts/c6403.ambari.apache.org", + "Hosts": { + "cpu_count": 6, + "total_mem": 409600, + "disk_info": [ + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"}, + {"mountpoint": "/"}, + {"mountpoint": "/dev/shm"}, + {"mountpoint": "/vagrant"} + ], + "public_host_name": "c6403.ambari.apache.org", + "host_name": "c6403.ambari.apache.org" + } + } + ] + } + + services = { + "Versions": { + "parent_stack_version": "2.5", + "stack_name": "HDP", + "stack_version": "2.6", + "stack_hierarchy": { + "stack_name": "HDP", + "stack_versions": ["2.5", "2.4", "2.3", "2.2", "2.1", "2.0.6"] + } + }, + "services": [{ + "StackServices": { + "service_name": "DRUID", + }, + "components": [ + { + "StackServiceComponents": { + "component_name": "DRUID_COORDINATOR", + "hostnames": ["c6401.ambari.apache.org"] + }, + }, + { + "StackServiceComponents": { + "component_name": "DRUID_OVERLORD", + "hostnames": ["c6401.ambari.apache.org"] + }, + }, + { + "StackServiceComponents": { + "component_name": "DRUID_BROKER", + "hostnames": ["c6402.ambari.apache.org", "c6403.ambari.apache.org"] + }, + }, + { + "StackServiceComponents": { + "component_name": "DRUID_HISTORICAL", + "hostnames": ["c6401.ambari.apache.org", "c6403.ambari.apache.org"] + }, + }, + { + "StackServiceComponents": { + "component_name": "DRUID_MIDDLEMANAGER", + "hostnames": ["c6401.ambari.apache.org"] + }, + } + ] + } + ], + "configurations": { + "druid-common": { + "properties": { + "database_name": "druid", + "metastore_hostname": "c6401.ambari.apache.org", + "druid.metadata.storage.type": "derby", + "druid.extensions.loadList": "[\"mysql-metadata-storage\"]", + "druid.extensions.pullList": "[]" + } + } + } + } + + clusterData = { + } + + configurations = { + } + + self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations, + {'druid-historical': { + 'properties': {'druid.processing.numThreads': '5', + 'druid.server.http.numThreads': '40', + 'druid.processing.numMergeBuffers': '2', + 'druid.processing.buffer.sizeBytes': '67108864'}}, + 'druid-broker': { + 'properties': {'druid.processing.numThreads': '3', + 'druid.server.http.numThreads': '40', + 'druid.processing.numMergeBuffers': '2', + 'druid.processing.buffer.sizeBytes': '67108864'}}, + 'druid-common': {'properties': {'druid.extensions.loadList': '[]', + 'druid.metadata.storage.connector.port': '1527', + 'druid.metadata.storage.connector.connectURI': 'jdbc:derby://c6401.ambari.apache.org:1527/druid;create=true', + 'druid.zk.service.host': '' + }}, + 'druid-env': {'properties': {}, + 'property_attributes': {'druid.coordinator.jvm.heap.memory': {'maximum': '1024'}, + 'druid.overlord.jvm.heap.memory': {'maximum': '1024'}, + 'druid.middlemanager.jvm.heap.memory': { + 'maximum': '1024'}, + 'druid.historical.jvm.heap.memory': {'maximum': '1024'}, + 'druid.broker.jvm.heap.memory': {'maximum': '1024'}}}} + ) + def test_recommendAtlasConfigurations(self): configurations = {
