AMBARI-5303 Some changes to OSCheck (aonishuk via dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9451ee05 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9451ee05 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9451ee05 Branch: refs/heads/trunk Commit: 9451ee052b9b0a060432ce76ab30d72afec41790 Parents: 233dd01 Author: Dmitry Sen <d...@hortonworks.com> Authored: Fri Apr 4 12:56:37 2014 +0300 Committer: Dmitry Sen <d...@hortonworks.com> Committed: Fri Apr 4 12:56:37 2014 +0300 ---------------------------------------------------------------------- ambari-agent/pom.xml | 13 +++++ .../src/main/python/ambari_agent/Facter.py | 6 +- .../src/main/python/ambari_agent/HostCleanup.py | 8 +-- .../src/main/python/ambari_agent/HostInfo.py | 15 ++--- .../python/resource_management/core/system.py | 8 +-- .../ambari_agent/TestActualConfigHandler.py | 7 ++- .../test/python/ambari_agent/TestAmbariAgent.py | 5 +- .../test/python/ambari_agent/TestController.py | 23 ++++++-- .../test/python/ambari_agent/TestHardware.py | 41 +++++++++++--- .../test/python/ambari_agent/TestHeartbeat.py | 15 +++-- .../test/python/ambari_agent/TestHostCleanup.py | 11 ++-- .../test/python/ambari_agent/TestHostInfo.py | 22 +++++--- .../src/test/python/ambari_agent/TestMain.py | 17 +++--- .../python/ambari_agent/TestRegistration.py | 18 +++--- .../test/python/ambari_agent/TestSecurity.py | 15 ++--- .../main/python/common_functions/os_check.py | 58 ++++++-------------- ambari-server/pom.xml | 39 ++++++++----- ambari-server/src/main/python/ambari-server.py | 21 ++++--- ambari-server/src/main/python/bootstrap.py | 5 +- ambari-server/src/main/python/os_check_type.py | 44 +++++++++++++++ ambari-server/src/test/python/TestBootstrap.py | 6 +- 21 files changed, 246 insertions(+), 151 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9451ee05/ambari-agent/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-agent/pom.xml b/ambari-agent/pom.xml index a685ef0..c400fe6 100644 --- a/ambari-agent/pom.xml +++ b/ambari-agent/pom.xml @@ -538,6 +538,19 @@ <filemode>755</filemode> </mapper> </data> + <data> + <src> + ${project.basedir}/../ambari-common/src/main/python/common_functions + </src> + <type>directory</type> + <mapper> + <type>perm</type> + <prefix>${common_functions.install.dir}</prefix> + <filemode>755</filemode> + <user>root</user> + <group>root</group> + </mapper> + </data> </dataSet> </configuration> </plugin> http://git-wip-us.apache.org/repos/asf/ambari/blob/9451ee05/ambari-agent/src/main/python/ambari_agent/Facter.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/Facter.py b/ambari-agent/src/main/python/ambari_agent/Facter.py index 552f0e0..f9bca94 100644 --- a/ambari-agent/src/main/python/ambari_agent/Facter.py +++ b/ambari-agent/src/main/python/ambari_agent/Facter.py @@ -111,11 +111,11 @@ class Facter(): # Returns the full name of the OS def getOperatingSystem(self): - return OSCheck().get_os_type() + return OSCheck.get_os_type() # Returns the OS version def getOperatingSystemRelease(self): - return OSCheck().get_os_version() + return OSCheck.get_os_version() # Returns the OS TimeZone def getTimeZone(self): @@ -151,7 +151,7 @@ class Facter(): # Returns the operating system family def getOsFamily(self): - return OSCheck().get_os_family() + return OSCheck.get_os_family() def isSeLinux(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/9451ee05/ambari-agent/src/main/python/ambari_agent/HostCleanup.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py index da16dec..13f9d34 100644 --- a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py +++ b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py @@ -286,7 +286,7 @@ class HostCleanup: def find_repo_files_for_repos(self, repoNames): repoFiles = [] - osType = self.get_os_type() + osType = OSCheck.get_os_family() repoNameList = [] for repoName in repoNames: if len(repoName.strip()) > 0: @@ -321,7 +321,7 @@ class HostCleanup: packageStr = ' '.join(packageList) logger.debug("Erasing packages: " + packageStr) if packageStr is not None and packageStr: - os_name = self.get_os_type() + os_name = OSCheck.get_os_family() command = '' if os_name == 'suse': command = PACKAGE_ERASE_CMD_SUSE.format(packageStr) @@ -411,10 +411,6 @@ class HostCleanup: def is_current_user_root(self): return os.getuid() == 0 - def get_os_type(self): - return OSCheck().get_os_family() - - # Run command as sudoer by default, if root no issues def run_os_command(self, cmd, runWithSudo=True): if runWithSudo: http://git-wip-us.apache.org/repos/asf/ambari/blob/9451ee05/ambari-agent/src/main/python/ambari_agent/HostInfo.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/HostInfo.py b/ambari-agent/src/main/python/ambari_agent/HostInfo.py index 81cf29a..725f24f 100644 --- a/ambari-agent/src/main/python/ambari_agent/HostInfo.py +++ b/ambari-agent/src/main/python/ambari_agent/HostInfo.py @@ -48,7 +48,7 @@ class HostInfo: # List of live services checked for on the host, takes a map of plan strings DEFAULT_LIVE_SERVICES = [ - {"redhat":"ntpd", "suse":"ntp", "ubuntu":"ntp"} + {"redhat":"ntpd", "suse":"ntp", "debian":"ntp"} ] # Set of default users (need to be replaced with the configured user names) @@ -103,13 +103,13 @@ class HostInfo: TIMEOUT_SECONDS = 60 RESULT_UNAVAILABLE = "unable_to_determine" - OS_NAME = OSCheck().get_os_family() - OS_UBUNTU = 'ubuntu' + OS_FAMILY = OSCheck.get_os_family() + OS_UBUNTU_DEBIAN = 'debian' # service cmd SERVICE_CMD = "/sbin/service" FIREWALL_SERVICE_NAME = "iptables" # on ubuntu iptables service is called ufw - if OS_NAME == OS_UBUNTU: + if OS_FAMILY == OS_UBUNTU_DEBIAN: SERVICE_CMD = "/usr/sbin/service" FIREWALL_SERVICE_NAME = "ufw" @@ -163,11 +163,8 @@ class HostInfo: result['target'] = realConf etcResults.append(result) - def get_os_type(self): - return OSCheck().get_os_family() - def checkLiveServices(self, services, result): - osType = self.get_os_type() + osType = OSCheck.get_os_family() for service in services: svcCheckResult = {} if isinstance(service, dict): @@ -313,7 +310,7 @@ class HostInfo: dict['umask'] = str(self.getUMask()) # detailed host check is not available for Suse - isSuse = 'suse' == self.get_os_type() + isSuse = 'suse' == OSCheck.get_os_family() dict['iptablesIsRunning'] = self.checkIptables() http://git-wip-us.apache.org/repos/asf/ambari/blob/9451ee05/ambari-agent/src/main/python/resource_management/core/system.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/resource_management/core/system.py b/ambari-agent/src/main/python/resource_management/core/system.py index cb4c766..90a2559 100644 --- a/ambari-agent/src/main/python/resource_management/core/system.py +++ b/ambari-agent/src/main/python/resource_management/core/system.py @@ -68,7 +68,7 @@ class System(object): In case cannot detect --> Fail """ - return OSCheck().get_os_version() + return OSCheck.get_os_version() @lazy_property def os_release_name(self): @@ -76,7 +76,7 @@ class System(object): For Ubuntu 12.04: precise """ - return OSCheck().get_os_release_name() + return OSCheck.get_os_release_name() @lazy_property def os_type(self): @@ -88,7 +88,7 @@ class System(object): In case cannot detect raises exception. """ - return OSCheck().get_os_type() + return OSCheck.get_os_type() @lazy_property def os_family(self): @@ -98,7 +98,7 @@ class System(object): In case cannot detect raises exception """ - return OSCheck().get_os_family() + return OSCheck.get_os_family() @lazy_property def ec2(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/9451ee05/ambari-agent/src/test/python/ambari_agent/TestActualConfigHandler.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestActualConfigHandler.py b/ambari-agent/src/test/python/ambari_agent/TestActualConfigHandler.py index 6e4973e..ae66ead 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestActualConfigHandler.py +++ b/ambari-agent/src/test/python/ambari_agent/TestActualConfigHandler.py @@ -19,12 +19,15 @@ limitations under the License. ''' import tempfile from unittest import TestCase -from ambari_agent.AmbariConfig import AmbariConfig -from ambari_agent.ActualConfigHandler import ActualConfigHandler import os import logging from mock.mock import patch +with patch("platform.linux_distribution", return_value = ('Suse','11','Final')): + from ambari_agent.AmbariConfig import AmbariConfig + from ambari_agent.ActualConfigHandler import ActualConfigHandler + + class TestActualConfigHandler(TestCase): logger = logging.getLogger() http://git-wip-us.apache.org/repos/asf/ambari/blob/9451ee05/ambari-agent/src/test/python/ambari_agent/TestAmbariAgent.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestAmbariAgent.py b/ambari-agent/src/test/python/ambari_agent/TestAmbariAgent.py index 330581e..a7ee104 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestAmbariAgent.py +++ b/ambari-agent/src/test/python/ambari_agent/TestAmbariAgent.py @@ -23,8 +23,9 @@ import subprocess import os import sys from mock.mock import MagicMock, patch, ANY -from ambari_agent.Controller import AGENT_AUTO_RESTART_EXIT_CODE -from ambari_agent import AmbariAgent +with patch("platform.linux_distribution", return_value = ('Suse','11','Final')): + from ambari_agent.Controller import AGENT_AUTO_RESTART_EXIT_CODE + from ambari_agent import AmbariAgent class TestAmbariAgent(unittest.TestCase): http://git-wip-us.apache.org/repos/asf/ambari/blob/9451ee05/ambari-agent/src/test/python/ambari_agent/TestController.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestController.py b/ambari-agent/src/test/python/ambari_agent/TestController.py index 63f5b22..1e7c868 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestController.py +++ b/ambari-agent/src/test/python/ambari_agent/TestController.py @@ -22,14 +22,19 @@ limitations under the License. import StringIO import ssl import unittest, threading -from ambari_agent import Controller, ActionQueue -from ambari_agent import hostname import sys -from ambari_agent.Controller import AGENT_AUTO_RESTART_EXIT_CODE from mock.mock import patch, MagicMock, call, Mock import logging +import platform from threading import Event +with patch("platform.linux_distribution", return_value = ('Suse','11','Final')): + from ambari_agent import Controller, ActionQueue + from ambari_agent import hostname + from ambari_agent.Controller import AGENT_AUTO_RESTART_EXIT_CODE + from common_functions import OSCheck + +@patch.object(platform, "linux_distribution", new = ('Suse','11','Final')) class TestController(unittest.TestCase): logger = logging.getLogger() @@ -130,9 +135,13 @@ class TestController(unittest.TestCase): @patch("urllib2.build_opener") @patch("urllib2.install_opener") @patch.object(Controller, "ActionQueue") - def test_run(self, ActionQueue_mock, installMock, buildMock): + @patch.object(OSCheck, "get_os_type") + @patch.object(OSCheck, "get_os_version") + def test_run(self, get_os_version_mock, get_os_type_mock, ActionQueue_mock, installMock, buildMock): aq = MagicMock() ActionQueue_mock.return_value = aq + get_os_type_mock.return_value = "suse" + get_os_version_mock.return_value = "11" buildMock.return_value = "opener" registerAndHeartbeat = MagicMock("registerAndHeartbeat") @@ -164,10 +173,14 @@ class TestController(unittest.TestCase): @patch("urllib2.build_opener") @patch("urllib2.install_opener") @patch.object(ActionQueue.ActionQueue, "run") - def test_repeatRegistration(self, + @patch.object(OSCheck, "get_os_type") + @patch.object(OSCheck, "get_os_version") + def test_repeatRegistration(self, get_os_version_mock, get_os_type_mock, run_mock, installMock, buildMock): registerAndHeartbeat = MagicMock(name="registerAndHeartbeat") + get_os_type_mock.return_value = "suse" + get_os_version_mock.return_value = "11" self.controller.registerAndHeartbeat = registerAndHeartbeat self.controller.run() http://git-wip-us.apache.org/repos/asf/ambari/blob/9451ee05/ambari-agent/src/test/python/ambari_agent/TestHardware.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestHardware.py b/ambari-agent/src/test/python/ambari_agent/TestHardware.py index 682b932..0675f67 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestHardware.py +++ b/ambari-agent/src/test/python/ambari_agent/TestHardware.py @@ -19,14 +19,23 @@ limitations under the License. ''' from unittest import TestCase -from ambari_agent import hostname -from ambari_agent.Hardware import Hardware from mock.mock import patch -from ambari_agent.Facter import Facter import unittest +import platform +with patch("platform.linux_distribution", return_value = ('Suse','11','Final')): + from ambari_agent import hostname + from ambari_agent.Hardware import Hardware + from ambari_agent.Facter import Facter + from common_functions import OSCheck + +@patch.object(platform,"linux_distribution", new = ('Suse','11','Final')) class TestHardware(TestCase): - def test_build(self): + @patch.object(OSCheck, "get_os_type") + @patch.object(OSCheck, "get_os_version") + def test_build(self, get_os_version_mock, get_os_type_mock): + get_os_type_mock.return_value = "suse" + get_os_version_mock.return_value = "11" hardware = Hardware() result = hardware.get() osdisks = hardware.osdisks() @@ -79,9 +88,13 @@ class TestHardware(TestCase): @patch.object(hostname,"hostname") @patch.object(Facter, "getFqdn") - def test_fqdnDomainHostname(self, facter_getFqdn_mock, hostname_mock): + @patch.object(OSCheck, "get_os_type") + @patch.object(OSCheck, "get_os_version") + def test_fqdnDomainHostname(self, get_os_version_mock, get_os_type_mock, facter_getFqdn_mock, hostname_mock): facter_getFqdn_mock.return_value = "ambari.apache.org" hostname_mock.return_value = 'ambari' + get_os_type_mock.return_value = "suse" + get_os_version_mock.return_value = "11" result = Facter().facterInfo() self.assertEquals(result['hostname'], "ambari") @@ -89,9 +102,13 @@ class TestHardware(TestCase): self.assertEquals(result['fqdn'], (result['hostname'] + '.' + result['domain'])) @patch.object(Facter, "setDataUpTimeOutput") - def test_uptimeSecondsHoursDays(self, facter_setDataUpTimeOutput_mock): + @patch.object(OSCheck, "get_os_type") + @patch.object(OSCheck, "get_os_version") + def test_uptimeSecondsHoursDays(self, get_os_version_mock, get_os_type_mock, facter_setDataUpTimeOutput_mock): # 3 days + 1 hour + 13 sec facter_setDataUpTimeOutput_mock.return_value = "262813.00 123.45" + get_os_type_mock.return_value = "suse" + get_os_version_mock.return_value = "11" result = Facter().facterInfo() self.assertEquals(result['uptime_seconds'], '262813') @@ -99,7 +116,9 @@ class TestHardware(TestCase): self.assertEquals(result['uptime_days'], '3') @patch.object(Facter, "setMemInfoOutput") - def test_facterMemInfoOutput(self, facter_setMemInfoOutput_mock): + @patch.object(OSCheck, "get_os_type") + @patch.object(OSCheck, "get_os_version") + def test_facterMemInfoOutput(self, get_os_version_mock, get_os_type_mock, facter_setMemInfoOutput_mock): facter_setMemInfoOutput_mock.return_value = ''' MemTotal: 1832392 kB @@ -112,6 +131,8 @@ SwapTotal: 2139592 kB SwapFree: 1598676 kB ''' + get_os_type_mock.return_value = "suse" + get_os_version_mock.return_value = "11" result = Facter().facterInfo() self.assertEquals(result['memorysize'], 1832392) @@ -121,7 +142,9 @@ SwapFree: 1598676 kB self.assertEquals(result['swapfree'], '1.52 GB') @patch.object(Facter, "setDataIfConfigOutput") - def test_facterDataIfConfigOutput(self, facter_setDataIfConfigOutput_mock): + @patch.object(OSCheck, "get_os_type") + @patch.object(OSCheck, "get_os_version") + def test_facterDataIfConfigOutput(self, get_os_version_mock, get_os_type_mock, facter_setDataIfConfigOutput_mock): facter_setDataIfConfigOutput_mock.return_value = ''' eth0 Link encap:Ethernet HWaddr 08:00:27:C9:39:9E @@ -152,6 +175,8 @@ lo Link encap:Local Loopback RX bytes:0 (0.0 b) TX bytes:0 (0.0 b) ''' + get_os_type_mock.return_value = "suse" + get_os_version_mock.return_value = "11" result = Facter().facterInfo() self.assertEquals(result['ipaddress'], '10.0.2.15') http://git-wip-us.apache.org/repos/asf/ambari/blob/9451ee05/ambari-agent/src/test/python/ambari_agent/TestHeartbeat.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestHeartbeat.py b/ambari-agent/src/test/python/ambari_agent/TestHeartbeat.py index c6a834d..4703af4 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestHeartbeat.py +++ b/ambari-agent/src/test/python/ambari_agent/TestHeartbeat.py @@ -20,19 +20,22 @@ limitations under the License. from unittest import TestCase import unittest -from ambari_agent.Heartbeat import Heartbeat -from ambari_agent.ActionQueue import ActionQueue -from ambari_agent.LiveStatus import LiveStatus -from ambari_agent import AmbariConfig import socket import os import time from mock.mock import patch, MagicMock, call -from ambari_agent.StackVersionsFileHandler import StackVersionsFileHandler -from ambari_agent.HostInfo import HostInfo import StringIO import sys +with patch("platform.linux_distribution", return_value = ('Suse','11','Final')): + from ambari_agent.Heartbeat import Heartbeat + from ambari_agent.ActionQueue import ActionQueue + from ambari_agent.LiveStatus import LiveStatus + from ambari_agent import AmbariConfig + from ambari_agent.StackVersionsFileHandler import StackVersionsFileHandler + from ambari_agent.HostInfo import HostInfo + + class TestHeartbeat(TestCase): def setUp(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/9451ee05/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py b/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py index b624212..34ee284 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py +++ b/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py @@ -28,6 +28,7 @@ import tempfile import os.path import optparse import logging +from common_functions import OSCheck PACKAGE_SECTION = "packages" PACKAGE_KEY = "pkg_list" @@ -174,7 +175,7 @@ class TestHostCleanup(TestCase): @patch.object(HostCleanup.HostCleanup, 'get_additional_dirs') @patch.object(HostCleanup.HostCleanup, 'do_erase_alternatives') @patch.object(HostCleanup.HostCleanup, 'find_repo_files_for_repos') - @patch.object(HostCleanup.HostCleanup, 'get_os_type') + @patch.object(OSCheck, "get_os_type") @patch.object(HostCleanup.HostCleanup, 'do_kill_processes') @patch.object(HostCleanup.HostCleanup, 'do_erase_files_silent') @patch.object(HostCleanup.HostCleanup, 'do_erase_dir_silent') @@ -220,7 +221,7 @@ class TestHostCleanup(TestCase): @patch.object(HostCleanup.HostCleanup, 'get_user_ids') @patch.object(HostCleanup.HostCleanup, 'do_erase_alternatives') @patch.object(HostCleanup.HostCleanup, 'find_repo_files_for_repos') - @patch.object(HostCleanup.HostCleanup, 'get_os_type') + @patch.object(OSCheck, "get_os_type") @patch.object(HostCleanup.HostCleanup, 'do_kill_processes') @patch.object(HostCleanup.HostCleanup, 'do_erase_files_silent') @patch.object(HostCleanup.HostCleanup, 'do_erase_dir_silent') @@ -260,7 +261,7 @@ class TestHostCleanup(TestCase): sys.stdout = sys.__stdout__ @patch.object(HostCleanup.HostCleanup, 'find_repo_files_for_repos') - @patch.object(HostCleanup.HostCleanup, 'get_os_type') + @patch.object(OSCheck, "get_os_type") @patch.object(HostCleanup.HostCleanup, 'do_kill_processes') @patch.object(HostCleanup.HostCleanup, 'do_erase_files_silent') @patch.object(HostCleanup.HostCleanup, 'do_erase_dir_silent') @@ -340,7 +341,7 @@ class TestHostCleanup(TestCase): @patch.object(HostCleanup.HostCleanup, 'run_os_command') - @patch.object(HostCleanup.HostCleanup, 'get_os_type') + @patch.object(OSCheck, "get_os_type") def test_do_earse_packages(self, get_os_type_method, run_os_command_method): out = StringIO.StringIO() sys.stdout = out @@ -373,7 +374,7 @@ class TestHostCleanup(TestCase): sys.stdout = sys.__stdout__ @patch.object(HostCleanup.HostCleanup, 'get_files_in_dir') - @patch.object(HostCleanup.HostCleanup, 'get_os_type') + @patch.object(OSCheck, "get_os_type") def test_find_repo_files_for_repos(self, get_os_type_method, get_files_in_dir_method): out = StringIO.StringIO() http://git-wip-us.apache.org/repos/asf/ambari/blob/9451ee05/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py index b199907..e76ccc4 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py +++ b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py @@ -18,6 +18,7 @@ See the License for the specific language governing permissions and limitations under the License. ''' + from unittest import TestCase import logging import unittest @@ -25,12 +26,15 @@ import subprocess from mock.mock import patch from mock.mock import MagicMock from mock.mock import create_autospec -from ambari_agent.HostCheckReportFileHandler import HostCheckReportFileHandler -from ambari_agent.PackagesAnalyzer import PackagesAnalyzer -from ambari_agent.HostInfo import HostInfo -from ambari_agent.Hardware import Hardware -from ambari_agent.AmbariConfig import AmbariConfig -from resource_management.core.system import System + +with patch("platform.linux_distribution", return_value = ('redhat','11','Final')): + from ambari_agent.HostCheckReportFileHandler import HostCheckReportFileHandler + from ambari_agent.PackagesAnalyzer import PackagesAnalyzer + from ambari_agent.HostInfo import HostInfo + from ambari_agent.Hardware import Hardware + from ambari_agent.AmbariConfig import AmbariConfig + from resource_management.core.system import System + from common_functions import OSCheck @patch.object(System, "os_family", new = 'redhat') class TestHostInfo(TestCase): @@ -232,7 +236,7 @@ class TestHostInfo(TestCase): self.assertTrue(newlist[1]['status'], "Invalid home directory") - @patch.object(HostInfo, 'get_os_type') + @patch.object(OSCheck, "get_os_type") @patch('os.umask') @patch.object(HostCheckReportFileHandler, 'writeHostCheckFile') @patch.object(PackagesAnalyzer, 'allAvailablePackages') @@ -272,7 +276,7 @@ class TestHostInfo(TestCase): self.assertTrue('agentTimeStampAtReporting' in dict['hostHealth']) - @patch.object(HostInfo, 'get_os_type') + @patch.object(OSCheck, "get_os_type") @patch('os.umask') @patch.object(HostCheckReportFileHandler, 'writeHostCheckFile') @patch.object(PackagesAnalyzer, 'allAvailablePackages') @@ -441,7 +445,7 @@ class TestHostInfo(TestCase): self.assertEquals(result, {}) - @patch.object(HostInfo, "get_os_type") + @patch.object(OSCheck, "get_os_type") @patch("subprocess.Popen") def test_checkLiveServices(self, subproc_popen, get_os_type_method): hostInfo = HostInfo() http://git-wip-us.apache.org/repos/asf/ambari/blob/9451ee05/ambari-agent/src/test/python/ambari_agent/TestMain.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestMain.py b/ambari-agent/src/test/python/ambari_agent/TestMain.py index 7dab39e..afe9b59 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestMain.py +++ b/ambari-agent/src/test/python/ambari_agent/TestMain.py @@ -19,22 +19,23 @@ limitations under the License. ''' import StringIO import sys - -from ambari_agent import NetUtil, security from mock.mock import MagicMock, patch, ANY import unittest -from ambari_agent import ProcessHelper, main -from ambari_agent import ProcessHelper, main import logging import signal -from ambari_agent.AmbariConfig import AmbariConfig import ConfigParser import os import tempfile -from ambari_agent.PingPortListener import PingPortListener -from ambari_agent.Controller import Controller from optparse import OptionParser -from ambari_agent.DataCleaner import DataCleaner + +with patch("platform.linux_distribution", return_value = ('Suse','11','Final')): + from ambari_agent import NetUtil, security + from ambari_agent import ProcessHelper, main + from ambari_agent import ProcessHelper, main + from ambari_agent.AmbariConfig import AmbariConfig + from ambari_agent.PingPortListener import PingPortListener + from ambari_agent.Controller import Controller + from ambari_agent.DataCleaner import DataCleaner class TestMain(unittest.TestCase): http://git-wip-us.apache.org/repos/asf/ambari/blob/9451ee05/ambari-agent/src/test/python/ambari_agent/TestRegistration.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestRegistration.py b/ambari-agent/src/test/python/ambari_agent/TestRegistration.py index e63976e..c4e074d 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestRegistration.py +++ b/ambari-agent/src/test/python/ambari_agent/TestRegistration.py @@ -23,19 +23,23 @@ import os import tempfile from mock.mock import patch from mock.mock import MagicMock -from ambari_agent.Register import Register -from ambari_agent.AmbariConfig import AmbariConfig -from ambari_agent.HostInfo import HostInfo -class TestRegistration(TestCase): +with patch("platform.linux_distribution", return_value = ('Suse','11','Final')): + from ambari_agent.Register import Register + from ambari_agent.AmbariConfig import AmbariConfig + from ambari_agent.HostInfo import HostInfo + from common_functions import OSCheck - @patch.object(HostInfo, 'get_os_type') - def test_registration_build(self, get_os_type_method): +class TestRegistration(TestCase): + @patch.object(OSCheck, "get_os_type") + @patch.object(OSCheck, "get_os_version") + def test_registration_build(self, get_os_version_mock, get_os_type_mock): config = AmbariConfig().getConfig() tmpdir = tempfile.gettempdir() config.set('agent', 'prefix', tmpdir) config.set('agent', 'current_ping_port', '33777') - get_os_type_method.return_value = 'redhat' + get_os_type_mock.return_value = "suse" + get_os_version_mock.return_value = "11" ver_file = os.path.join(tmpdir, "version") with open(ver_file, "w") as text_file: text_file.write("1.3.0") http://git-wip-us.apache.org/repos/asf/ambari/blob/9451ee05/ambari-agent/src/test/python/ambari_agent/TestSecurity.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestSecurity.py b/ambari-agent/src/test/python/ambari_agent/TestSecurity.py index 9ef0d0c..db4b25e 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestSecurity.py +++ b/ambari-agent/src/test/python/ambari_agent/TestSecurity.py @@ -19,22 +19,23 @@ limitations under the License. ''' import StringIO import sys, subprocess - -from ambari_agent import NetUtil -from ambari_agent.security import CertificateManager from mock.mock import MagicMock, patch, ANY import mock.mock import unittest -from ambari_agent import ProcessHelper, main import logging import signal -from ambari_agent.AmbariConfig import AmbariConfig import ConfigParser import ssl import os import tempfile -from ambari_agent.Controller import Controller -from ambari_agent import security + +with patch("platform.linux_distribution", return_value = ('Suse','11','Final')): + from ambari_agent import NetUtil + from ambari_agent.security import CertificateManager + from ambari_agent import ProcessHelper, main + from ambari_agent.AmbariConfig import AmbariConfig + from ambari_agent.Controller import Controller + from ambari_agent import security aa = mock.mock.mock_open() class TestSecurity(unittest.TestCase): http://git-wip-us.apache.org/repos/asf/ambari/blob/9451ee05/ambari-common/src/main/python/common_functions/os_check.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/common_functions/os_check.py b/ambari-common/src/main/python/common_functions/os_check.py index e6591b5..982d1d0 100644 --- a/ambari-common/src/main/python/common_functions/os_check.py +++ b/ambari-common/src/main/python/common_functions/os_check.py @@ -19,15 +19,13 @@ limitations under the License. ''' import os -import sys import platform -class OSCheck(object): - def __init__(self): - pass +class OSCheck: - def get_os_type(self): + @staticmethod + def get_os_type(): """ Return values: redhat, fedora, centos, oraclelinux, ascendos, @@ -52,18 +50,17 @@ class OSCheck(object): if operatingSystem != '': return operatingSystem else: - print "Cannot detect os type. Exiting..." - sys.exit(1) + raise Exception("Cannot detect os type. Exiting...") - - def get_os_family(self): + @staticmethod + def get_os_family(): """ Return values: redhat, debian, suse ... and others In case cannot detect raises exception( from self.get_operating_system_type() ). """ - os_family = self.get_os_type() + os_family = OSCheck.get_os_type() if os_family in ['redhat', 'fedora', 'centos', 'oraclelinux', 'ascendos', 'amazon', 'xenserver', 'oel', 'ovs', 'cloudlinux', 'slc', 'scientific', 'psbm', 'centos linux']: @@ -72,11 +69,11 @@ class OSCheck(object): os_family = 'Debian' elif os_family in ['sles', 'sled', 'opensuse', 'suse']: os_family = 'Suse' - #else: os_family = self.get_os_type() + #else: os_family = OSCheck.get_os_type() return os_family.lower() - - def get_os_version(self): + @staticmethod + def get_os_version(): """ Returns the OS version @@ -90,18 +87,19 @@ class OSCheck(object): if dist: return dist else: - print "Cannot detect os version. Exiting..." - sys.exit(1) + raise Exception("Cannot detect os version. Exiting...") - def get_os_major_version(self): + @staticmethod + def get_os_major_version(): """ Returns the main OS version like Centos 6.5 --> 6 RedHat 1.2.3 --> 1 """ - return self.get_os_version().split('.')[0] + return OSCheck.get_os_version().split('.')[0] - def get_os_release_name(self): + @staticmethod + def get_os_release_name(): """ Returns the OS release name @@ -113,29 +111,7 @@ class OSCheck(object): if dist: return dist else: - print "Cannot detect os release name. Exiting..." - sys.exit(1) - - -def main(argv=None): - # Same logic that was in "os_type_check.sh" - if len(sys.argv) != 2: - print "Usage: <cluster_os>" - sys.exit(2) - pass - - cluster_os = sys.argv[1] - current_os = OSCheck().get_os_family() + OSCheck().get_os_major_version() + raise Exception("Cannot detect os release name. Exiting...") - # If agent/server have the same {"family","main_version"} - then ok. - print "Cluster primary/cluster OS type is %s and local/current OS type is %s" % ( - cluster_os, current_os) - if current_os == cluster_os: - sys.exit(0) - else: - print "Local OS is not compatible with cluster primary OS. Please perform manual bootstrap on this host." - sys.exit(1) -if __name__ == "__main__": - main() http://git-wip-us.apache.org/repos/asf/ambari/blob/9451ee05/ambari-server/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml index 02c2379..fafab1c 100644 --- a/ambari-server/pom.xml +++ b/ambari-server/pom.xml @@ -244,16 +244,16 @@ </source> </sources> </mapping> - <mapping> - <directory>${common_functions.install.dir}</directory> - <sources> - <source> - <location> - ${project.basedir}/../ambari-common/src/main/python/common_functions - </location> - </source> - </sources> - </mapping> + <mapping> + <directory>${common_functions.install.dir}</directory> + <sources> + <source> + <location> + ${project.basedir}/../ambari-common/src/main/python/common_functions + </location> + </source> + </sources> + </mapping> <mapping> <directory>/usr/sbin</directory> <filemode>755</filemode> @@ -417,7 +417,7 @@ <location>src/main/python/setupAgent.py</location> </source> <source> - <location>${project.basedir}/../ambari-common/src/main/python/common_functions/os_check.py</location> + <location>src/main/python/os_check_type.py</location> </source> </sources> </mapping> @@ -760,11 +760,11 @@ </mapper> </data> <data> - <src>${project.basedir}/../ambari-common/src/main/python/common_functions/os_check.py</src> + <src>src/main/python/os_check_type.py</src> <type>file</type> <mapper> <type>perm</type> - <prefix>/usr/lib/python2.6/site-packages/common_functions</prefix> + <prefix>/usr/lib/python2.6/site-packages/ambari_server</prefix> <user>root</user> <group>root</group> <filemode>755</filemode> @@ -797,6 +797,19 @@ <filemode>755</filemode> </mapper> </data> + <data> + <src> + ${project.basedir}/../ambari-common/src/main/python/common_functions + </src> + <type>directory</type> + <mapper> + <type>perm</type> + <prefix>${common_functions.install.dir}</prefix> + <filemode>755</filemode> + <user>root</user> + <group>root</group> + </mapper> + </data> </dataSet> </configuration> </plugin> http://git-wip-us.apache.org/repos/asf/ambari/blob/9451ee05/ambari-server/src/main/python/ambari-server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari-server.py b/ambari-server/src/main/python/ambari-server.py index ad1ec3c..72ae9b4 100755 --- a/ambari-server/src/main/python/ambari-server.py +++ b/ambari-server/src/main/python/ambari-server.py @@ -51,7 +51,8 @@ SERVER_START_DEBUG = False # OS info OS_VERSION = OSCheck().get_os_major_version() -OS = OSCheck().get_os_type() +OS_TYPE = OSCheck.get_os_type() +OS_FAMILY = OSCheck.get_os_family() # action commands SETUP_ACTION = "setup" @@ -242,11 +243,11 @@ CHANGE_OWNER_COMMAND = ['su', '-', 'postgres', '--command=/var/lib/ambari-server/resources/scripts/change_owner.sh -d {0} -s {1} -o {2}'] PG_ERROR_BLOCKED = "is being accessed by other users" -PG_STATUS_RUNNING = utils.get_postgre_running_status(OS) +PG_STATUS_RUNNING = utils.get_postgre_running_status(OS_TYPE) PG_DEFAULT_PASSWORD = "bigdata" SERVICE_CMD = "/usr/bin/env service" PG_SERVICE_NAME = "postgresql" -PG_HBA_DIR = utils.get_postgre_hba_dir(OS) +PG_HBA_DIR = utils.get_postgre_hba_dir(OS_TYPE) PG_ST_CMD = "%s %s status" % (SERVICE_CMD, PG_SERVICE_NAME) if os.path.isfile("/usr/bin/postgresql-setup"): @@ -493,11 +494,11 @@ class OpenSuseFirewallChecks(FirewallChecks): def get_firewall_object(): - if OS == utils.OS_UBUNTU: + if OS_TYPE == utils.OS_UBUNTU: return UbuntuFirewallChecks() - elif OS == utils.OS_FEDORA and int(OS_VERSION) >= 18: + elif OS_TYPE == utils.OS_FEDORA and int(OS_VERSION) >= 18: return Fedora18FirewallChecks() - elif OS == utils.OS_OPENSUSE: + elif OS_TYPE == utils.OS_OPENSUSE: return OpenSuseFirewallChecks() else: return FirewallChecks() @@ -1056,7 +1057,7 @@ def check_postgre_up(): return 0 else: # run initdb only on non ubuntu systems as ubuntu does not have initdb cmd. - if OS != utils.OS_UBUNTU: + if OS_TYPE != utils.OS_UBUNTU: print "Running initdb: This may take upto a minute." retcode, out, err = run_os_command(PG_INITDB_CMD) if retcode == 0: @@ -1068,7 +1069,7 @@ def check_postgre_up(): stdin=subprocess.PIPE, stderr=subprocess.PIPE ) - if OS == utils.OS_SUSE: + if OS_TYPE == utils.OS_SUSE: time.sleep(20) result = process.poll() print_info_msg("Result of postgres start cmd: " + str(result)) @@ -1985,9 +1986,7 @@ def configure_os_settings(): print_error_msg("Non-Linux systems are not supported") return -1 - os_name = OSCheck().get_os_family() - os_version = OS_VERSION - master_os_type = os_name + os_version + master_os_type = OS_FAMILY + OS_VERSION write_property(OS_TYPE_PROPERTY, master_os_type) return 0 http://git-wip-us.apache.org/repos/asf/ambari/blob/9451ee05/ambari-server/src/main/python/bootstrap.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/bootstrap.py b/ambari-server/src/main/python/bootstrap.py index ff7bf33..a25a0fe 100755 --- a/ambari-server/src/main/python/bootstrap.py +++ b/ambari-server/src/main/python/bootstrap.py @@ -37,6 +37,7 @@ MAX_PARALLEL_BOOTSTRAPS = 20 # How many seconds to wait between polling parallel bootstraps POLL_INTERVAL_SEC = 1 DEBUG=False +PYTHON_ENV="env PYTHONPATH=$PYTHONPATH:/usr/lib/python2.6/site-packages " class HostLog: @@ -135,7 +136,7 @@ class SSH: class Bootstrap(threading.Thread): """ Bootstrap the agent on a separate host""" TEMP_FOLDER = "/tmp" - OS_CHECK_SCRIPT_FILENAME = "os_check.py" + OS_CHECK_SCRIPT_FILENAME = "os_check_type.py" AMBARI_REPO_FILENAME = "ambari.repo" SETUP_SCRIPT_FILENAME = "setupAgent.py" PASSWORD_FILENAME = "host_pass" @@ -328,7 +329,7 @@ class Bootstrap(threading.Thread): self.host_log.write("Running OS type check...") command = "chmod a+x %s && %s %s" % \ (self.getOsCheckScriptRemoteLocation(), - self.getOsCheckScriptRemoteLocation(), params.cluster_os_type) + PYTHON_ENV + self.getOsCheckScriptRemoteLocation(), params.cluster_os_type) ssh = SSH(params.user, params.sshkey_file, self.host, command, params.bootdir, self.host_log) http://git-wip-us.apache.org/repos/asf/ambari/blob/9451ee05/ambari-server/src/main/python/os_check_type.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/os_check_type.py b/ambari-server/src/main/python/os_check_type.py new file mode 100644 index 0000000..ce3d9da --- /dev/null +++ b/ambari-server/src/main/python/os_check_type.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python2.6 + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import sys +from common_functions import OSCheck + +def main(argv=None): + # Same logic that was in "os_type_check.sh" + if len(sys.argv) != 2: + print "Usage: <cluster_os>" + raise Exception("Error in number of arguments. Usage: <cluster_os>") + pass + + cluster_os = sys.argv[1] + current_os = OSCheck.get_os_family() + OSCheck.get_os_major_version() + + # If agent/server have the same {"family","main_version"} - then ok. + print "Cluster primary/cluster OS type is %s and local/current OS type is %s" % ( + cluster_os, current_os) + if current_os == cluster_os: + sys.exit(0) + else: + raise Exception("Local OS is not compatible with cluster primary OS. Please perform manual bootstrap on this host.") + + +if __name__ == "__main__": + main() http://git-wip-us.apache.org/repos/asf/ambari/blob/9451ee05/ambari-server/src/test/python/TestBootstrap.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestBootstrap.py b/ambari-server/src/test/python/TestBootstrap.py index 76e6e43..f8f05a6 100644 --- a/ambari-server/src/test/python/TestBootstrap.py +++ b/ambari-server/src/test/python/TestBootstrap.py @@ -317,7 +317,7 @@ class TestBootstrap(TestCase): None, "8440") bootstrap_obj = Bootstrap("hostname", shared_state) ocs = bootstrap_obj.getOsCheckScript() - self.assertEquals(ocs, "scriptDir/os_check.py") + self.assertEquals(ocs, "scriptDir/os_check_type.py") @patch.object(Bootstrap, "getRemoteName") @@ -326,7 +326,7 @@ class TestBootstrap(TestCase): "setupAgentFile", "ambariServer", "centos6", None, "8440") bootstrap_obj = Bootstrap("hostname", shared_state) - v = "/tmp/os_check1374259902.py" + v = "/tmp/os_check_type1374259902.py" getRemoteName_mock.return_value = v ocs = bootstrap_obj.getOsCheckScriptRemoteLocation() self.assertEquals(ocs, v) @@ -459,7 +459,7 @@ class TestBootstrap(TestCase): self.assertEquals(res, expected) command = str(init_mock.call_args[0][3]) self.assertEqual(command, "chmod a+x OsCheckScriptRemoteLocation &&" - " OsCheckScriptRemoteLocation centos6") + " env PYTHONPATH=$PYTHONPATH:/usr/lib/python2.6/site-packages OsCheckScriptRemoteLocation centos6") @patch.object(SSH, "__init__")