AMBARI-22191. Account for merge buffers while calculating druid processing buffer size. (nishantmonu51)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/85fb356c Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/85fb356c Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/85fb356c Branch: refs/heads/feature-branch-AMBARI-21307 Commit: 85fb356ccd13669359810e767da14d014018bac9 Parents: c100197 Author: Nishant <nishant.mon...@gmail.com> Authored: Thu Oct 12 02:58:58 2017 +0530 Committer: Nishant <nishant.mon...@gmail.com> Committed: Thu Oct 12 02:58:58 2017 +0530 ---------------------------------------------------------------------- .../resources/stacks/HDP/2.6/services/stack_advisor.py | 2 +- .../test/python/stacks/2.6/common/test_stack_advisor.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/85fb356c/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py index 38af525..94f28db 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py @@ -159,7 +159,7 @@ class HDP26StackAdvisor(HDP25StackAdvisor): putComponentProperty('druid.processing.numMergeBuffers', numMergeBuffers) totalAvailableMemInMb = self.getMinMemory(component_hosts) / 1024 maxAvailableBufferSizeInMb = totalAvailableMemInMb/(processingThreads + numMergeBuffers) - putComponentProperty('druid.processing.buffer.sizeBytes', self.getDruidProcessingBufferSizeInMb(totalAvailableMemInMb) * 1024 * 1024) + putComponentProperty('druid.processing.buffer.sizeBytes', self.getDruidProcessingBufferSizeInMb(maxAvailableBufferSizeInMb) * 1024 * 1024) # returns the recommended druid processing buffer size in Mb. http://git-wip-us.apache.org/repos/asf/ambari/blob/85fb356c/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py index 1c937b7..ade08c1 100644 --- a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py @@ -985,7 +985,7 @@ class TestHDP26StackAdvisor(TestCase): "href": "/api/v1/hosts/c6402.ambari.apache.org", "Hosts": { "cpu_count": 1, - "total_mem": 1922680, + "total_mem": 622680, "disk_info": [ {"mountpoint": "/"}, {"mountpoint": "/dev/shm"}, @@ -1091,12 +1091,12 @@ class TestHDP26StackAdvisor(TestCase): 'properties': {'druid.processing.numThreads': '2', 'druid.server.http.numThreads': '40', 'druid.processing.numMergeBuffers': '2', - 'druid.processing.buffer.sizeBytes': '536870912'}}, + 'druid.processing.buffer.sizeBytes': '134217728'}}, 'druid-broker': { 'properties': {'druid.processing.numThreads': '1', 'druid.server.http.numThreads': '40', 'druid.processing.numMergeBuffers': '2', - 'druid.processing.buffer.sizeBytes': '268435456'}}, + 'druid.processing.buffer.sizeBytes': '67108864'}}, 'druid-common': {'properties': {'druid.extensions.loadList': '[]', 'druid.metadata.storage.connector.port': '1527', 'druid.metadata.storage.connector.connectURI': 'jdbc:derby://c6401.ambari.apache.org:1527/druid;create=true', @@ -1108,7 +1108,7 @@ class TestHDP26StackAdvisor(TestCase): 'druid.middlemanager.jvm.heap.memory': { 'maximum': '49152'}, 'druid.historical.jvm.heap.memory': {'maximum': '3755'}, - 'druid.broker.jvm.heap.memory': {'maximum': '1877'}}}} + 'druid.broker.jvm.heap.memory': {'maximum': '1024'}}}} ) def test_recommendDruidConfigurations_low_mem_hosts(self): @@ -1240,12 +1240,12 @@ class TestHDP26StackAdvisor(TestCase): 'properties': {'druid.processing.numThreads': '5', 'druid.server.http.numThreads': '40', 'druid.processing.numMergeBuffers': '2', - 'druid.processing.buffer.sizeBytes': '67108864'}}, + 'druid.processing.buffer.sizeBytes': '14680064'}}, 'druid-broker': { 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40', 'druid.processing.numMergeBuffers': '2', - 'druid.processing.buffer.sizeBytes': '67108864'}}, + 'druid.processing.buffer.sizeBytes': '41943040'}}, 'druid-common': {'properties': {'druid.extensions.loadList': '[]', 'druid.metadata.storage.connector.port': '1527', 'druid.metadata.storage.connector.connectURI': 'jdbc:derby://c6401.ambari.apache.org:1527/druid;create=true',
