Repository: ambari Updated Branches: refs/heads/trunk 24ac93332 -> 0f8bdd7c9
AMBARI-8925. Configs: OS specific defaults for min users (dlysnichenko) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0f8bdd7c Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0f8bdd7c Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0f8bdd7c Branch: refs/heads/trunk Commit: 0f8bdd7c97266cc762e2e85b6fa4f9d1781f94c1 Parents: 24ac933 Author: Lisnichenko Dmitro <[email protected]> Authored: Fri Dec 26 18:42:37 2014 +0200 Committer: Lisnichenko Dmitro <[email protected]> Committed: Fri Dec 26 18:43:04 2014 +0200 ---------------------------------------------------------------------- .../stacks/HDP/2.0.6/services/stack_advisor.py | 38 ++++++++++++++++++++ .../stacks/2.0.6/common/test_stack_advisor.py | 32 +++++++++++++++++ .../stacks/2.2/common/test_stack_advisor.py | 31 ++++++++++++++++ 3 files changed, 101 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/0f8bdd7c/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index ca3e51d..e88c2c5 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -23,6 +23,7 @@ from math import ceil from stack_advisor import DefaultStackAdvisor + class HDP206StackAdvisor(DefaultStackAdvisor): def getComponentLayoutValidations(self, services, hosts): @@ -93,9 +94,11 @@ class HDP206StackAdvisor(DefaultStackAdvisor): def recommendYARNConfigurations(self, configurations, clusterData, services, hosts): putYarnProperty = self.putProperty(configurations, "yarn-site") + putYarnEnvProperty = self.putProperty(configurations, "yarn-env") putYarnProperty('yarn.nodemanager.resource.memory-mb', int(round(clusterData['containers'] * clusterData['ramPerContainer']))) putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['ramPerContainer'])) putYarnProperty('yarn.scheduler.maximum-allocation-mb', int(round(clusterData['containers'] * clusterData['ramPerContainer']))) + putYarnEnvProperty('min_user_id', self.get_system_min_uid()) def recommendMapReduce2Configurations(self, configurations, clusterData, services, hosts): putMapredProperty = self.putProperty(configurations, "mapred-site") @@ -321,6 +324,41 @@ class HDP206StackAdvisor(DefaultStackAdvisor): 'WEBHCAT_SERVER': {6: 1, 31: 2, "else": 4}, } + def get_system_min_uid(self): + login_defs = '/etc/login.defs' + uid_min_tag = 'UID_MIN' + comment_tag = '#' + uid_min = uid_default = '1000' + uid = None + + if os.path.exists(login_defs): + with open(login_defs, 'r') as f: + data = f.read().split('\n') + # look for uid_min_tag in file + uid = filter(lambda x: uid_min_tag in x, data) + # filter all lines, where uid_min_tag was found in comments + uid = filter(lambda x: x.find(comment_tag) > x.find(uid_min_tag) or x.find(comment_tag) == -1, uid) + + if uid is not None and len(uid) > 0: + uid = uid[0] + comment = uid.find(comment_tag) + tag = uid.find(uid_min_tag) + if comment == -1: + uid_tag = tag + len(uid_min_tag) + uid_min = uid[uid_tag:].strip() + elif comment > tag: + uid_tag = tag + len(uid_min_tag) + uid_min = uid[uid_tag:comment].strip() + + # check result for value + try: + int(uid_min) + except ValueError: + return uid_default + + return uid_min + + # Validation helper methods def getSiteProperties(configurations, siteName): siteConfig = configurations.get(siteName) http://git-wip-us.apache.org/repos/asf/ambari/blob/0f8bdd7c/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py index f192c5b..fc5f66b 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py @@ -18,6 +18,7 @@ limitations under the License. import socket from unittest import TestCase +from mock.mock import patch, MagicMock class TestHDP206StackAdvisor(TestCase): @@ -36,6 +37,32 @@ class TestHDP206StackAdvisor(TestCase): clazz = getattr(stack_advisor_impl, hdp206StackAdvisorClassName) self.stackAdvisor = clazz() + # substitute method in the instance + self.get_system_min_uid_real = self.stackAdvisor.get_system_min_uid + self.stackAdvisor.get_system_min_uid = self.get_system_min_uid_magic + + @patch('__builtin__.open') + @patch('os.path.exists') + def get_system_min_uid_magic(self, exists_mock, open_mock): + class MagicFile(object): + def read(self): + return """ + #test line UID_MIN 200 + UID_MIN 500 + """ + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + + def __enter__(self): + return self + + exists_mock.return_value = True + open_mock.return_value = MagicFile() + return self.get_system_min_uid_real() + + + def test_recommendationCardinalityALL(self): servicesInfo = [ { @@ -328,6 +355,11 @@ class TestHDP206StackAdvisor(TestCase): "ramPerContainer": 256 } expected = { + "yarn-env": { + "properties": { + "min_user_id": "500" + } + }, "yarn-site": { "properties": { "yarn.nodemanager.resource.memory-mb": "1280", http://git-wip-us.apache.org/repos/asf/ambari/blob/0f8bdd7c/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 49e2629..4bdd52a 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -18,6 +18,8 @@ limitations under the License. import os from unittest import TestCase +from mock.mock import patch, MagicMock + class TestHDP22StackAdvisor(TestCase): @@ -41,6 +43,30 @@ class TestHDP22StackAdvisor(TestCase): clazz = getattr(stack_advisor_impl, hdp22StackAdvisorClassName) self.stackAdvisor = clazz() + # substitute method in the instance + self.get_system_min_uid_real = self.stackAdvisor.get_system_min_uid + self.stackAdvisor.get_system_min_uid = self.get_system_min_uid_magic + + @patch('__builtin__.open') + @patch('os.path.exists') + def get_system_min_uid_magic(self, exists_mock, open_mock): + class MagicFile(object): + def read(self): + return """ + #test line UID_MIN 200 + UID_MIN 500 + """ + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + + def __enter__(self): + return self + + exists_mock.return_value = True + open_mock.return_value = MagicFile() + return self.get_system_min_uid_real() + def test_recommendTezConfigurations(self): configurations = {} clusterData = { @@ -484,6 +510,11 @@ class TestHDP22StackAdvisor(TestCase): "ramPerContainer": 256 } expected = { + "yarn-env": { + "properties": { + "min_user_id": "500" + } + }, "yarn-site": { "properties": { "yarn.nodemanager.resource.memory-mb": "1280",
