This is an automated email from the ASF dual-hosted git repository. aonishuk pushed a commit to branch branch-2.7 in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-2.7 by this push: new 347c61a AMBARI-24782. Introduce support for Ubuntu 18 LTS (aonishuk) 347c61a is described below commit 347c61a71bcab605b1c05f7f95e5704ac27527f9 Author: Andrew Onishuk <aonis...@hortonworks.com> AuthorDate: Tue Oct 16 13:35:41 2018 +0300 AMBARI-24782. Introduce support for Ubuntu 18 LTS (aonishuk) --- .../src/main/package/dependencies.properties | 2 +- .../resource_management/TestRepositoryResource.py | 16 +++---- .../python/ambari_commons/resources/os_family.json | 3 +- .../libraries/providers/repository.py | 49 ++++++++++++++++------ .../resource_management/libraries/script/script.py | 7 +++- .../src/main/resources/version_definition.xsd | 1 + 6 files changed, 54 insertions(+), 24 deletions(-) diff --git a/ambari-agent/src/main/package/dependencies.properties b/ambari-agent/src/main/package/dependencies.properties index 07b0b68..ec64264 100644 --- a/ambari-agent/src/main/package/dependencies.properties +++ b/ambari-agent/src/main/package/dependencies.properties @@ -29,4 +29,4 @@ # however should be encouraged manually in pom.xml. rpm.dependency.list=openssl,\nRequires: rpm-python,\nRequires: zlib,\nRequires: python >= 2.6 -deb.dependency.list=openssl, zlibc, python (>= 2.6) \ No newline at end of file +deb.dependency.list=openssl, python (>= 2.6) \ No newline at end of file diff --git a/ambari-agent/src/test/python/resource_management/TestRepositoryResource.py b/ambari-agent/src/test/python/resource_management/TestRepositoryResource.py index a69b57b..b1a4757 100644 --- a/ambari-agent/src/test/python/resource_management/TestRepositoryResource.py +++ b/ambari-agent/src/test/python/resource_management/TestRepositoryResource.py @@ -188,7 +188,7 @@ class TestRepositoryResource(TestCase): @patch.object(OSCheck, "is_suse_family") @patch.object(OSCheck, "is_ubuntu_family") @patch.object(OSCheck, "is_redhat_family") - @patch("resource_management.libraries.providers.repository.checked_call") + @patch("resource_management.libraries.providers.repository.call") @patch.object(tempfile, "NamedTemporaryFile") @patch("resource_management.libraries.providers.repository.Execute") @patch("resource_management.libraries.providers.repository.File") @@ -197,13 +197,13 @@ class TestRepositoryResource(TestCase): @patch.object(System, "os_release_name", new='precise') @patch.object(System, "os_family", new='ubuntu') def test_create_repo_ubuntu_repo_exists(self, file_mock, execute_mock, - tempfile_mock, checked_call_mock, is_redhat_family, is_ubuntu_family, is_suse_family): + tempfile_mock, call_mock, is_redhat_family, is_ubuntu_family, is_suse_family): is_redhat_family.return_value = False is_ubuntu_family.return_value = True is_suse_family.return_value = False tempfile_mock.return_value = MagicMock(spec=file) tempfile_mock.return_value.__enter__.return_value.name = "/tmp/1.txt" - checked_call_mock.return_value = 0, "The following signatures couldn't be verified because the public key is not available: NO_PUBKEY 123ABCD" + call_mock.return_value = 0, "The following signatures couldn't be verified because the public key is not available: NO_PUBKEY 123ABCD" with Environment('/') as env: with patch.object(repository, "__file__", new='/ambari/test/repo/dummy/path/file'): @@ -228,10 +228,10 @@ class TestRepositoryResource(TestCase): #'apt-get update -qq -o Dir::Etc::sourcelist="sources.list.d/HDP.list" -o APT::Get::List-Cleanup="0"') execute_command_item = execute_mock.call_args_list[0][0][0] - self.assertEqual(checked_call_mock.call_args_list[0][0][0], ['apt-get', 'update', '-qq', '-o', 'Dir::Etc::sourcelist=sources.list.d/HDP.list', '-o', 'Dir::Etc::sourceparts=-', '-o', 'APT::Get::List-Cleanup=0']) + self.assertEqual(call_mock.call_args_list[0][0][0], ['apt-get', 'update', '-qq', '-o', 'Dir::Etc::sourcelist=sources.list.d/HDP.list', '-o', 'Dir::Etc::sourceparts=-', '-o', 'APT::Get::List-Cleanup=0']) self.assertEqual(execute_command_item, ('apt-key', 'adv', '--recv-keys', '--keyserver', 'keyserver.ubuntu.com', '123ABCD')) - @patch("resource_management.libraries.providers.repository.checked_call") + @patch("resource_management.libraries.providers.repository.call") @patch.object(tempfile, "NamedTemporaryFile") @patch("resource_management.libraries.providers.repository.Execute") @patch("resource_management.libraries.providers.repository.File") @@ -240,13 +240,13 @@ class TestRepositoryResource(TestCase): @patch.object(System, "os_release_name", new='precise') @patch.object(System, "os_family", new='ubuntu') def test_create_repo_ubuntu_gpg_key_wrong_output(self, file_mock, execute_mock, - tempfile_mock, checked_call_mock): + tempfile_mock, call_mock): """ Checks that GPG key is extracted from output without \r sign """ tempfile_mock.return_value = MagicMock(spec=file) tempfile_mock.return_value.__enter__.return_value.name = "/tmp/1.txt" - checked_call_mock.return_value = 0, "The following signatures couldn't be verified because the public key is not available: NO_PUBKEY 123ABCD\r\n" + call_mock.return_value = 0, "The following signatures couldn't be verified because the public key is not available: NO_PUBKEY 123ABCD\r\n" with Environment('/') as env: with patch.object(repository, "__file__", new='/ambari/test/repo/dummy/path/file'): @@ -270,7 +270,7 @@ class TestRepositoryResource(TestCase): self.assertEqual(copy_item1, "call('/etc/apt/sources.list.d/HDP.list', content=StaticFile('/tmp/1.txt'))") execute_command_item = execute_mock.call_args_list[0][0][0] - self.assertEqual(checked_call_mock.call_args_list[0][0][0], ['apt-get', 'update', '-qq', '-o', 'Dir::Etc::sourcelist=sources.list.d/HDP.list', '-o', 'Dir::Etc::sourceparts=-', '-o', 'APT::Get::List-Cleanup=0']) + self.assertEqual(call_mock.call_args_list[0][0][0], ['apt-get', 'update', '-qq', '-o', 'Dir::Etc::sourcelist=sources.list.d/HDP.list', '-o', 'Dir::Etc::sourceparts=-', '-o', 'APT::Get::List-Cleanup=0']) self.assertEqual(execute_command_item, ('apt-key', 'adv', '--recv-keys', '--keyserver', 'keyserver.ubuntu.com', '123ABCD')) @patch.object(tempfile, "NamedTemporaryFile") diff --git a/ambari-common/src/main/python/ambari_commons/resources/os_family.json b/ambari-common/src/main/python/ambari_commons/resources/os_family.json index ca7614f..0634988 100644 --- a/ambari-common/src/main/python/ambari_commons/resources/os_family.json +++ b/ambari-common/src/main/python/ambari_commons/resources/os_family.json @@ -58,7 +58,8 @@ "versions": [ 12, 14, - 16 + 16, + 18 ] }, "suse": { diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/repository.py b/ambari-common/src/main/python/resource_management/libraries/providers/repository.py index f476e29..bf4799e 100644 --- a/ambari-common/src/main/python/resource_management/libraries/providers/repository.py +++ b/ambari-common/src/main/python/resource_management/libraries/providers/repository.py @@ -31,9 +31,10 @@ from resource_management.core.source import InlineTemplate from resource_management.core.source import StaticFile from resource_management.libraries.functions.format import format from resource_management.core.environment import Environment -from resource_management.core.shell import checked_call +from resource_management.core.shell import checked_call, call from resource_management.core import sudo from resource_management.core.logger import Logger +from resource_management.core.exceptions import ExecutionFailed import re REPO_TEMPLATE_FOLDER = 'data' @@ -119,18 +120,42 @@ class UbuntuRepositoryProvider(Provider): content = StaticFile(tmpf.name) ) - update_cmd_formatted = [format(x) for x in self.update_cmd] - # this is time expensive - retcode, out = checked_call(update_cmd_formatted, sudo=True, quiet=False) - - # add public keys for new repos - missing_pkeys = set(re.findall(self.missing_pkey_regex, out)) - for pkey in missing_pkeys: - Execute(self.app_pkey_cmd_prefix + (pkey,), - timeout = 15, # in case we are on the host w/o internet (using localrepo), we should ignore hanging - ignore_failures = True, - sudo = True, + try: + self.update(repo_file_path) + except: + # remove created file or else ambari will consider that update was successful and skip repository operations + File(repo_file_path, + action = "delete", ) + raise + + def update(self, repo_file_path): + repo_file_name = os.path.basename(repo_file_path) + update_cmd_formatted = [format(x) for x in self.update_cmd] + + update_failed_exception = None + + try: + # this is time expensive + retcode, out = call(update_cmd_formatted, sudo=True, quiet=False) + except ExecutionFailed as ex: + out = ex.out + update_failed_exception = ex + + # add public keys for new repos + missing_pkeys = set(re.findall(self.missing_pkey_regex, out)) + + # failed but NOT due to missing pubkey + if update_failed_exception and not missing_pkeys: + raise update_failed_exception + + for pkey in missing_pkeys: + # add public keys for new repos + Execute(self.app_pkey_cmd_prefix + (pkey,), + timeout = 15, # in case we are on the host w/o internet (using localrepo), we should ignore hanging + ignore_failures = True, + sudo = True, + ) def action_remove(self): repo_file_name = format("{repo_file_name}.list", repo_file_name=self.resource.repo_file_name) diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index 2a17208..9d99025 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -358,8 +358,11 @@ class Script(object): ex.pre_raise() raise finally: - if self.should_expose_component_version(self.command_name): - self.save_component_version_to_structured_out(self.command_name) + try: + if self.should_expose_component_version(self.command_name): + self.save_component_version_to_structured_out(self.command_name) + except: + Logger.exception("Reporting component version failed") def get_version(self, env): pass diff --git a/ambari-server/src/main/resources/version_definition.xsd b/ambari-server/src/main/resources/version_definition.xsd index 5f806fc..fb38370 100644 --- a/ambari-server/src/main/resources/version_definition.xsd +++ b/ambari-server/src/main/resources/version_definition.xsd @@ -60,6 +60,7 @@ <xs:enumeration value="ubuntu12" /> <xs:enumeration value="ubuntu14" /> <xs:enumeration value="ubuntu16" /> + <xs:enumeration value="ubuntu18" /> <xs:enumeration value="suse11" /> <xs:enumeration value="suse12" /> <xs:enumeration value="amazonlinux2" />