Repository: ambari Updated Branches: refs/heads/branch-2.2 3c0c22c17 -> 6ed6758e6
AMBARI-15007. Make amazon2015 to be part of redhat6 family (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6ed6758e Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6ed6758e Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6ed6758e Branch: refs/heads/branch-2.2 Commit: 6ed6758e620da2315107075eb76e26a1272040b1 Parents: 3c0c22c Author: Andrew Onishuk <aonis...@hortonworks.com> Authored: Fri Feb 12 12:06:37 2016 +0200 Committer: Andrew Onishuk <aonis...@hortonworks.com> Committed: Fri Feb 12 12:06:37 2016 +0200 ---------------------------------------------------------------------- .../src/main/python/ambari_commons/os_check.py | 73 +++++++--- .../ambari_commons/resources/os_family.json | 137 +++++++++---------- .../server/state/stack/JsonOsFamilyRoot.java | 38 +++++ .../ambari/server/state/stack/OsFamily.java | 8 +- .../resources/stacks/HDP/2.2/repos/repoinfo.xml | 12 -- .../resources/stacks/HDP/2.3/repos/repoinfo.xml | 12 -- .../resources/stacks/HDP/2.4/repos/repoinfo.xml | 12 -- ambari-server/src/test/python/TestOSCheck.py | 37 +++-- ambari-server/src/test/resources/os_family.json | 89 ++++++------ 9 files changed, 239 insertions(+), 179 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/6ed6758e/ambari-common/src/main/python/ambari_commons/os_check.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_commons/os_check.py b/ambari-common/src/main/python/ambari_commons/os_check.py index 238c276..d30b7f9 100644 --- a/ambari-common/src/main/python/ambari_commons/os_check.py +++ b/ambari-common/src/main/python/ambari_commons/os_check.py @@ -56,6 +56,8 @@ RESOURCES_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "resou # family JSON data OSFAMILY_JSON_RESOURCE = "os_family.json" +JSON_OS_MAPPING = "mapping" +JSON_OS_ALIASES = "aliases" JSON_OS_TYPE = "distro" JSON_OS_VERSION = "versions" JSON_EXTENDS = "extends" @@ -76,6 +78,8 @@ VER_NT_SERVER = 3 _IS_ORACLE_LINUX = os.path.exists('/etc/oracle-release') _IS_REDHAT_LINUX = os.path.exists('/etc/redhat-release') +SYSTEM_RELEASE_FILE = "/etc/system-release" + def _is_oracle_linux(): return _IS_ORACLE_LINUX @@ -84,16 +88,16 @@ def _is_redhat_linux(): def advanced_check(distribution): distribution = list(distribution) - if os.path.exists("/etc/issue"): - with open("/etc/issue", "rb") as fp: + if os.path.exists(SYSTEM_RELEASE_FILE): + with open(SYSTEM_RELEASE_FILE, "rb") as fp: issue_content = fp.read() if "Amazon" in issue_content: distribution[0] = "amazon" - search_groups = re.search('(\d+)\.(\d+)', issue_content) + search_groups = re.search('(\d+\.\d+)', issue_content) if search_groups: - distribution[1] = search_groups.group(1) # if version is 2015.09 only get 2015. + distribution[1] = search_groups.group(1) return tuple(distribution) @@ -114,16 +118,24 @@ class OS_CONST_TYPE(type): f = open(os.path.join(RESOURCES_DIR, OSFAMILY_JSON_RESOURCE)) json_data = eval(f.read()) f.close() - for family in json_data: + + if JSON_OS_MAPPING not in json_data: + raise Exception("Invalid {0}".format(OSFAMILY_JSON_RESOURCE)) + + json_mapping_data = json_data[JSON_OS_MAPPING] + + for family in json_mapping_data: cls.FAMILY_COLLECTION += [family] - cls.OS_COLLECTION += json_data[family][JSON_OS_TYPE] + cls.OS_COLLECTION += json_mapping_data[family][JSON_OS_TYPE] cls.OS_FAMILY_COLLECTION += [{ 'name': family, - 'os_list': json_data[family][JSON_OS_TYPE] + 'os_list': json_mapping_data[family][JSON_OS_TYPE] }] - if JSON_EXTENDS in json_data[family]: - cls.OS_FAMILY_COLLECTION[-1][JSON_EXTENDS] = json_data[family][JSON_EXTENDS] + if JSON_EXTENDS in json_mapping_data[family]: + cls.OS_FAMILY_COLLECTION[-1][JSON_EXTENDS] = json_mapping_data[family][JSON_EXTENDS] + + cls.OS_TYPE_ALIASES = json_data[JSON_OS_ALIASES] if JSON_OS_ALIASES in json_data else {} except: raise Exception("Couldn't load '%s' file" % OSFAMILY_JSON_RESOURCE) @@ -194,7 +206,24 @@ class OSCheck: distribution = ("Darwin", "TestOnly", "1.1.1", "1.1.1", "1.1") return distribution - + + @staticmethod + def get_alias(os_type, os_version): + version_parts = os_version.split('.') + full_os_and_major_version = os_type + version_parts[0] + + if full_os_and_major_version in OSConst.OS_TYPE_ALIASES: + alias = OSConst.OS_TYPE_ALIASES[full_os_and_major_version] + re_groups = re.search('(\D+)(\d+)$', alias).groups() + os_type = re_groups[0] + os_major_version = re_groups[1] + + version_parts[0] = os_major_version + os_version = '.'.join(version_parts) + + return os_type, os_version + + @staticmethod def get_os_type(): """ @@ -205,6 +234,10 @@ class OSCheck: In case cannot detect - exit. """ + return OSCheck.get_alias(OSCheck._get_os_type(), OSCheck._get_os_version())[0] + + @staticmethod + def _get_os_type(): # Read content from /etc/*-release file # Full release name dist = OSCheck.os_distribution() @@ -212,18 +245,18 @@ class OSCheck: # special cases if _is_oracle_linux(): - return 'oraclelinux' + operatingSystem = 'oraclelinux' elif operatingSystem.startswith('suse linux enterprise server'): - return 'sles' + operatingSystem = 'sles' elif operatingSystem.startswith('red hat enterprise linux'): - return 'redhat' + operatingSystem = 'redhat' elif operatingSystem.startswith('darwin'): - return 'mac' + operatingSystem = 'mac' - if operatingSystem != '': - return operatingSystem - else: + if operatingSystem == '': raise Exception("Cannot detect os type. Exiting...") + + return operatingSystem @staticmethod def get_os_family(): @@ -257,11 +290,15 @@ class OSCheck: In case cannot detect raises exception. """ + return OSCheck.get_alias(OSCheck._get_os_type(), OSCheck._get_os_version())[1] + + @staticmethod + def _get_os_version(): # Read content from /etc/*-release file # Full release name dist = OSCheck.os_distribution() dist = dist[1] - + if dist: return dist else: http://git-wip-us.apache.org/repos/asf/ambari/blob/6ed6758e/ambari-common/src/main/python/ambari_commons/resources/os_family.json ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_commons/resources/os_family.json b/ambari-common/src/main/python/ambari_commons/resources/os_family.json index 13014fc..1558c1b 100644 --- a/ambari-common/src/main/python/ambari_commons/resources/os_family.json +++ b/ambari-common/src/main/python/ambari_commons/resources/os_family.json @@ -1,72 +1,69 @@ { - "redhat": { - "distro": [ - "redhat", - "fedora", - "centos", - "oraclelinux", - "ascendos", - "xenserver", - "oel", - "ovs", - "cloudlinux", - "slc", - "scientific", - "psbm", - "centos linux" - ], - "versions": [ - 6, - 7 - ] - }, - "amazon": { - "extends" : "redhat", - "distro": [ - "amazon" - ], - "versions": [ - 2015 - ] - }, - "debian": { - "extends" : "ubuntu", - "distro": [ - "debian" - ], - "versions": [ - 7 - ] - }, - "ubuntu": { - "distro": [ - "ubuntu" - ], - "versions": [ - 12, - 14 - ] - }, - "suse": { - "distro": [ - "sles", - "sled", - "opensuse", - "suse" - ], - "versions": [ - 11 - ] - }, - "winsrv": { - "distro": [ - "win2008server", - "win2008serverr2", - "win2012server", - "win2012serverr2" - ], - "versions": [ - 6 - ] - } + "mapping": { + "redhat": { + "distro": [ + "redhat", + "fedora", + "centos", + "oraclelinux", + "amazon", + "ascendos", + "xenserver", + "oel", + "ovs", + "cloudlinux", + "slc", + "scientific", + "psbm", + "centos linux" + ], + "versions": [ + 6, + 7 + ] + }, + "debian": { + "extends" : "ubuntu", + "distro": [ + "debian" + ], + "versions": [ + 7 + ] + }, + "ubuntu": { + "distro": [ + "ubuntu" + ], + "versions": [ + 12, + 14 + ] + }, + "suse": { + "distro": [ + "sles", + "sled", + "opensuse", + "suse" + ], + "versions": [ + 11 + ] + }, + "winsrv": { + "distro": [ + "win2008server", + "win2008serverr2", + "win2012server", + "win2012serverr2" + ], + "versions": [ + 6 + ] + } + }, + "aliases": { + "amazon2015": "amazon6" + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/6ed6758e/ambari-server/src/main/java/org/apache/ambari/server/state/stack/JsonOsFamilyRoot.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/JsonOsFamilyRoot.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/JsonOsFamilyRoot.java new file mode 100644 index 0000000..3f9158f --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/JsonOsFamilyRoot.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.state.stack; + +import java.util.Map; + +public class JsonOsFamilyRoot { + private Map<String, JsonOsFamilyEntry> mapping; + private Map<String, String> aliases; + + public Map<String, JsonOsFamilyEntry> getMapping() { + return mapping; + } + public void setMapping(Map<String, JsonOsFamilyEntry> mapping) { + this.mapping = mapping; + } + public Map<String, String> getAliases() { + return aliases; + } + public void setAliases(Map<String, String> aliases) { + this.aliases = aliases; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/6ed6758e/ambari-server/src/main/java/org/apache/ambari/server/state/stack/OsFamily.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/OsFamily.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/OsFamily.java index 37a6db3..e494c44 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/OsFamily.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/OsFamily.java @@ -48,11 +48,14 @@ public class OsFamily { private final String os_pattern = "([\\D]+|(?:[\\D]+[\\d]+[\\D]+))([\\d]*)"; private final String OS_DISTRO = "distro"; private final String OS_VERSION = "versions"; + private final String OS_MAPPING = "mapping"; + private final String OS_ALIASES = "aliases"; private final String LOAD_CONFIG_MSG = "Could not load OS family definition from %s file"; private final String FILE_NAME = "os_family.json"; private final Logger LOG = LoggerFactory.getLogger(OsFamily.class); private Map<String, JsonOsFamilyEntry> osMap = null; + private JsonOsFamilyRoot jsonOsFamily = null; /** * Initialize object @@ -77,9 +80,10 @@ public class OsFamily { if (!f.exists()) throw new Exception(); inputStream = new FileInputStream(f); - Type type = new TypeToken<Map<String, JsonOsFamilyEntry>>() {}.getType(); + Type type = new TypeToken<JsonOsFamilyRoot>() {}.getType(); Gson gson = new Gson(); - osMap = gson.fromJson(new InputStreamReader(inputStream), type); + jsonOsFamily = gson.fromJson(new InputStreamReader(inputStream), type); + osMap = jsonOsFamily.getMapping(); } catch (Exception e) { LOG.error(String.format(LOAD_CONFIG_MSG, new File(SharedResourcesPath, FILE_NAME).toString())); throw new RuntimeException(e); http://git-wip-us.apache.org/repos/asf/ambari/blob/6ed6758e/ambari-server/src/main/resources/stacks/HDP/2.2/repos/repoinfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/repos/repoinfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/repos/repoinfo.xml index 9decf51..dbf8506 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/repos/repoinfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/repos/repoinfo.xml @@ -29,18 +29,6 @@ <reponame>HDP-UTILS</reponame> </repo> </os> - <os family="amazon2015"> - <repo> - <baseurl>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.2.6.0</baseurl> - <repoid>HDP-2.2</repoid> - <reponame>HDP</reponame> - </repo> - <repo> - <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6</baseurl> - <repoid>HDP-UTILS-1.1.0.20</repoid> - <reponame>HDP-UTILS</reponame> - </repo> - </os> <os family="suse11"> <repo> <baseurl>http://public-repo-1.hortonworks.com/HDP/suse11sp3/2.x/updates/2.2.6.0</baseurl> http://git-wip-us.apache.org/repos/asf/ambari/blob/6ed6758e/ambari-server/src/main/resources/stacks/HDP/2.3/repos/repoinfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/repos/repoinfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/repos/repoinfo.xml index 279134b..142b87d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/repos/repoinfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/repos/repoinfo.xml @@ -41,18 +41,6 @@ <reponame>HDP-UTILS</reponame> </repo> </os> - <os family="amazon2015"> - <repo> - <baseurl>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.0.0</baseurl> - <repoid>HDP-2.3</repoid> - <reponame>HDP</reponame> - </repo> - <repo> - <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6</baseurl> - <repoid>HDP-UTILS-1.1.0.20</repoid> - <reponame>HDP-UTILS</reponame> - </repo> - </os> <os family="suse11"> <repo> <baseurl>http://public-repo-1.hortonworks.com/HDP/suse11sp3/2.x/updates/2.3.0.0</baseurl> http://git-wip-us.apache.org/repos/asf/ambari/blob/6ed6758e/ambari-server/src/main/resources/stacks/HDP/2.4/repos/repoinfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/repos/repoinfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/repos/repoinfo.xml index 42dae37..54bd3da 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/repos/repoinfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/repos/repoinfo.xml @@ -41,18 +41,6 @@ <reponame>HDP-UTILS</reponame> </repo> </os> - <os family="amazon2015"> - <repo> - <baseurl>http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/updates/2.4.0.0</baseurl> - <repoid>HDP-2.4</repoid> - <reponame>HDP</reponame> - </repo> - <repo> - <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6</baseurl> - <repoid>HDP-UTILS-1.1.0.20</repoid> - <reponame>HDP-UTILS</reponame> - </repo> - </os> <os family="suse11"> <repo> <baseurl>http://s3.amazonaws.com/dev.hortonworks.com/HDP/suse11sp3/2.x/updates/2.4.0.0</baseurl> http://git-wip-us.apache.org/repos/asf/ambari/blob/6ed6758e/ambari-server/src/test/python/TestOSCheck.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestOSCheck.py b/ambari-server/src/test/python/TestOSCheck.py index ebef706..e14458b 100644 --- a/ambari-server/src/test/python/TestOSCheck.py +++ b/ambari-server/src/test/python/TestOSCheck.py @@ -28,7 +28,7 @@ from unittest import TestCase from mock.mock import patch from mock.mock import MagicMock -from ambari_commons import OSCheck +from ambari_commons import OSCheck, OSConst import os_check_type utils = __import__('ambari_server.utils').utils @@ -48,7 +48,7 @@ class TestOSCheck(TestCase): # 1 - Any system mock_is_oracle_linux.return_value = False - mock_linux_distribution.return_value = ('my_os', '', '') + mock_linux_distribution.return_value = ('my_os', '2015.09', '') result = OSCheck.get_os_type() self.assertEquals(result, 'my_os') @@ -64,13 +64,13 @@ class TestOSCheck(TestCase): # 3 - path exist: '/etc/oracle-release' mock_is_oracle_linux.return_value = True - mock_linux_distribution.return_value = ('some_os', '', '') + mock_linux_distribution.return_value = ('some_os', '1234', '') result = OSCheck.get_os_type() self.assertEquals(result, 'oraclelinux') # 4 - Common system mock_is_oracle_linux.return_value = False - mock_linux_distribution.return_value = ('CenToS', '', '') + mock_linux_distribution.return_value = ('CenToS', '4.56', '') result = OSCheck.get_os_type() self.assertEquals(result, 'centos') @@ -97,31 +97,31 @@ class TestOSCheck(TestCase): # 1 - Any system mock_exists.return_value = False - mock_linux_distribution.return_value = ('MY_os', '', '') + mock_linux_distribution.return_value = ('MY_os', '5.6.7', '') result = OSCheck.get_os_family() self.assertEquals(result, 'my_os') # 2 - Redhat mock_exists.return_value = False - mock_linux_distribution.return_value = ('Centos Linux', '', '') + mock_linux_distribution.return_value = ('Centos Linux', '2.4', '') result = OSCheck.get_os_family() self.assertEquals(result, 'redhat') # 3 - Ubuntu mock_exists.return_value = False - mock_linux_distribution.return_value = ('Ubuntu', '', '') + mock_linux_distribution.return_value = ('Ubuntu', '14.04', '') result = OSCheck.get_os_family() self.assertEquals(result, 'ubuntu') # 4 - Suse mock_exists.return_value = False mock_linux_distribution.return_value = ( - 'suse linux enterprise server', '', '') + 'suse linux enterprise server', '11.3', '') result = OSCheck.get_os_family() self.assertEquals(result, 'suse') mock_exists.return_value = False - mock_linux_distribution.return_value = ('SLED', '', '') + mock_linux_distribution.return_value = ('SLED', '1.2.3.4.5', '') result = OSCheck.get_os_family() self.assertEquals(result, 'suse') @@ -139,7 +139,7 @@ class TestOSCheck(TestCase): def test_get_os_version(self, mock_linux_distribution): # 1 - Any system - mock_linux_distribution.return_value = ('', '123.45', '') + mock_linux_distribution.return_value = ('some_os', '123.45', '') result = OSCheck.get_os_version() self.assertEquals(result, '123.45') @@ -157,7 +157,7 @@ class TestOSCheck(TestCase): def test_get_os_major_version(self, mock_linux_distribution): # 1 - mock_linux_distribution.return_value = ('', '123.45.67', '') + mock_linux_distribution.return_value = ('abcd_os', '123.45.67', '') result = OSCheck.get_os_major_version() self.assertEquals(result, '123') @@ -165,6 +165,21 @@ class TestOSCheck(TestCase): mock_linux_distribution.return_value = ('Suse', '11', '') result = OSCheck.get_os_major_version() self.assertEquals(result, '11') + + @patch.object(OSCheck, "os_distribution") + def test_aliases(self, mock_linux_distribution): + OSConst.OS_TYPE_ALIASES['qwerty_os123'] = 'aliased_os5' + OSConst.OS_FAMILY_COLLECTION.append({ + 'name': 'aliased_os_family', + 'os_list': ["aliased_os"] + }) + + mock_linux_distribution.return_value = ('qwerty_os', '123.45.67', '') + + self.assertEquals(OSCheck.get_os_type(), 'aliased_os') + self.assertEquals(OSCheck.get_os_major_version(), '5') + self.assertEquals(OSCheck.get_os_version(), '5.45.67') + self.assertEquals(OSCheck.get_os_family(), 'aliased_os_family') @patch.object(OSCheck, "os_distribution") def test_get_os_release_name(self, mock_linux_distribution): http://git-wip-us.apache.org/repos/asf/ambari/blob/6ed6758e/ambari-server/src/test/resources/os_family.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/os_family.json b/ambari-server/src/test/resources/os_family.json index df55b61..ae6b19e 100644 --- a/ambari-server/src/test/resources/os_family.json +++ b/ambari-server/src/test/resources/os_family.json @@ -1,45 +1,50 @@ { - "redhat": { - "distro": [ - "redhat", - "fedora", - "centos", - "oraclelinux" - ], - "versions": [ - 5, - 6 - ] + "mapping": { + "redhat": { + "distro": [ + "redhat", + "fedora", + "centos", + "oraclelinux" + ], + "versions": [ + 5, + 6 + ] + }, + "ubuntu": { + "distro": [ + "ubuntu", + "debian" + ], + "versions": [ + 12 + ] + }, + "suse": { + "distro": [ + "sles", + "sled", + "opensuse", + "suse" + ], + "versions": [ + 11 + ] + }, + "winsrv": { + "distro": [ + "win2008server", + "win2008serverr2", + "win2012server", + "win2012serverr2" + ], + "versions": [ + 6 + ] + } }, - "ubuntu": { - "distro": [ - "ubuntu", - "debian" - ], - "versions": [ - 12 - ] - }, - "suse": { - "distro": [ - "sles", - "sled", - "opensuse", - "suse" - ], - "versions": [ - 11 - ] - }, - "winsrv": { - "distro": [ - "win2008server", - "win2008serverr2", - "win2012server", - "win2012serverr2" - ], - "versions": [ - 6 - ] + "aliases": { + "amazon2015": "amazon6" } -} +} \ No newline at end of file