AMBARI-9951. hadooplzo_2_2_2_0_2538-native can not be upgraded during RU (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8ce17b99 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8ce17b99 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8ce17b99 Branch: refs/heads/branch-2.0.0 Commit: 8ce17b991e7bdaba82622efbfb2b0fc888618707 Parents: 1956e0c Author: Lisnichenko Dmitro <[email protected]> Authored: Fri Mar 6 11:57:59 2015 +0200 Committer: Lisnichenko Dmitro <[email protected]> Committed: Fri Mar 6 12:03:20 2015 +0200 ---------------------------------------------------------------------- .../custom_actions/scripts/install_packages.py | 18 +++- .../custom_actions/TestInstallPackages.py | 91 ++++++++++++++++---- .../configs/install_packages_config.json | 4 +- 3 files changed, 91 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8ce17b99/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py index 5819390..eb93e18 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py @@ -29,6 +29,7 @@ from resource_management.libraries.functions.list_ambari_managed_repos import li from ambari_commons.os_check import OSCheck, OSConst from resource_management.libraries.functions.packages_analyzer import allInstalledPackages from resource_management.core.shell import call +from resource_management.libraries.functions.default import default class InstallPackages(Script): @@ -204,13 +205,24 @@ class InstallPackages(Script): """ filtered_package_list = [] for package in package_list: + skip_package = False # mysql* package logic is managed at HIVE scripts if package['name'].startswith('mysql'): - continue + skip_package = True # Ambari metrics packages should not be upgraded during RU if package['name'].startswith('ambari-metrics'): - continue - filtered_package_list.append(package) + skip_package = True + # hadooplzo package is installed only if LZO comperession is enabled + io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) + if not io_compression_codecs or "com.hadoop.compression.lzo" not in io_compression_codecs: + lzo_packages = ['hadoop-lzo', 'lzo', 'hadoop-lzo-native', 'liblzo2-2', 'hadooplzo'] + for lzo_package in lzo_packages: + if package['name'].startswith(lzo_package): + skip_package = True + break + + if not skip_package: + filtered_package_list.append(package) return filtered_package_list if __name__ == "__main__": http://git-wip-us.apache.org/repos/asf/ambari/blob/8ce17b99/ambari-server/src/test/python/custom_actions/TestInstallPackages.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/custom_actions/TestInstallPackages.py b/ambari-server/src/test/python/custom_actions/TestInstallPackages.py index 4975757..d2c340b 100644 --- a/ambari-server/src/test/python/custom_actions/TestInstallPackages.py +++ b/ambari-server/src/test/python/custom_actions/TestInstallPackages.py @@ -63,7 +63,7 @@ class TestInstallPackages(RMFTestCase): 'stack_id': 'HDP-2.2', 'ambari_repositories': []}) self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885', - base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0', + base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0', action=['create'], components=[u'HDP-UTILS', 'main'], repo_template='repo_suse_rhel.j2', @@ -72,7 +72,7 @@ class TestInstallPackages(RMFTestCase): append_to_file=False, ) self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885', - base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0', + base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0', action=['create'], components=[u'HDP', 'main'], repo_template='repo_suse_rhel.j2', @@ -83,8 +83,6 @@ class TestInstallPackages(RMFTestCase): self.assertResourceCalled('Package', 'hadoop_2_2_*', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885']) self.assertResourceCalled('Package', 'snappy', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885']) self.assertResourceCalled('Package', 'snappy-devel', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885']) - self.assertResourceCalled('Package', 'lzo', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885']) - self.assertResourceCalled('Package', 'hadooplzo_2_2_*', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885']) self.assertResourceCalled('Package', 'hadoop_2_2_*-libhdfs', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885']) self.assertResourceCalled('Package', 'ambari-log4j', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885']) self.assertNoMoreResources() @@ -113,7 +111,7 @@ class TestInstallPackages(RMFTestCase): 'stack_id': 'HDP-2.2', 'ambari_repositories': ["HDP-UTILS-2.2.0.1-885"]}) self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885', - base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0', + base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0', action=['create'], components=[u'HDP-UTILS', 'main'], repo_template=u'repo_suse_rhel.j2', @@ -122,7 +120,7 @@ class TestInstallPackages(RMFTestCase): append_to_file=False, ) self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885', - base_url='http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0', + base_url='http://repo1/HDP/centos5/2.x/updates/2.2.0.0', action=['create'], components=[u'HDP', 'main'], repo_template=u'repo_suse_rhel.j2', @@ -133,8 +131,6 @@ class TestInstallPackages(RMFTestCase): self.assertResourceCalled('Package', 'hadoop_2_2_*', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885']) self.assertResourceCalled('Package', 'snappy', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885']) self.assertResourceCalled('Package', 'snappy-devel', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885']) - self.assertResourceCalled('Package', 'lzo', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885']) - self.assertResourceCalled('Package', 'hadooplzo_2_2_*', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885']) self.assertResourceCalled('Package', 'hadoop_2_2_*-libhdfs', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885']) self.assertResourceCalled('Package', 'ambari-log4j', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885']) self.assertNoMoreResources() @@ -186,7 +182,7 @@ class TestInstallPackages(RMFTestCase): self.assertEquals(put_structured_out_mock.call_args[0][0], {'stack_id': 'HDP-2.2', 'installed_repository_version': u'2.2.0.1-885', 'ambari_repositories': [], 'package_installation_result': 'FAIL'}) self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885', - base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0', + base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0', action=['create'], components=[u'HDP-UTILS', 'main'], repo_template=u'repo_suse_rhel.j2', @@ -195,7 +191,7 @@ class TestInstallPackages(RMFTestCase): append_to_file=False, ) self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885', - base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0', + base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0', action=['create'], components=[u'HDP', 'main'], repo_template=u'repo_suse_rhel.j2', @@ -230,7 +226,7 @@ class TestInstallPackages(RMFTestCase): 'stack_id': 'HDP-2.2', 'ambari_repositories': []}) self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885', - base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0', + base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0', action=['create'], components=[u'HDP-UTILS', 'main'], repo_template='repo_suse_rhel.j2', @@ -239,7 +235,7 @@ class TestInstallPackages(RMFTestCase): append_to_file=False, ) self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885', - base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0', + base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0', action=['create'], components=[u'HDP', 'main'], repo_template='repo_suse_rhel.j2', @@ -250,8 +246,6 @@ class TestInstallPackages(RMFTestCase): self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885*', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885']) self.assertResourceCalled('Package', 'snappy', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885']) self.assertResourceCalled('Package', 'snappy-devel', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885']) - self.assertResourceCalled('Package', 'lzo', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885']) - self.assertResourceCalled('Package', 'hadooplzo_2_2_0_1_885*', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885']) self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885*-libhdfs', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885']) self.assertResourceCalled('Package', 'ambari-log4j', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885']) self.assertNoMoreResources() @@ -272,7 +266,8 @@ class TestInstallPackages(RMFTestCase): config_dict['roleParams']['package_list'] = '[{\"name\":\"mysql-connector-java\"},{\"name\":\"hive_2_2_*\"},' \ '{\"name\":\"hive_2_2_*-hcatalog\"},{\"name\":\"hive_2_2_*-webhcat\"},' \ - '{\"name\":\"mysql\"},{\"name\":\"mysql-client\"}, {\"name\":\"ambari-metrics-collector\"}]' + '{\"name\":\"mysql\"},{\"name\":\"mysql-client\"}, {\"name\":\"ambari-metrics-collector\"},' \ + ' {\"name\":\"hadooplzo_2_2_2_0_2538-native\"}]' self.executeScript("scripts/install_packages.py", classname="InstallPackages", @@ -288,7 +283,7 @@ class TestInstallPackages(RMFTestCase): 'stack_id': 'HDP-2.2', 'ambari_repositories': []}) self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885', - base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0', + base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0', action=['create'], components=[u'HDP-UTILS', 'main'], repo_template='repo_suse_rhel.j2', @@ -297,7 +292,7 @@ class TestInstallPackages(RMFTestCase): append_to_file=False, ) self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885', - base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0', + base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0', action=['create'], components=[u'HDP', 'main'], repo_template='repo_suse_rhel.j2', @@ -319,3 +314,65 @@ class TestInstallPackages(RMFTestCase): ) self.assertNoMoreResources() + # Check case when LZO is enabled + config_file = self._getSrcFolder()+"/test/python/custom_actions/configs/install_packages_config.json" + with open(config_file, "r") as f: + config_dict = json.load(f) + config_dict['configurations']['core-site'] = { + 'io.compression.codecs': "org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec," + "org.apache.hadoop.io.compress.SnappyCodec,com.hadoop.compression.lzo" + } + config_dict['roleParams']['package_list'] = '[{\"name\":\"mysql-connector-java\"},{\"name\":\"hive_2_2_*\"},' \ + '{\"name\":\"hive_2_2_*-hcatalog\"},{\"name\":\"hive_2_2_*-webhcat\"},' \ + '{\"name\":\"mysql\"},{\"name\":\"mysql-client\"}, {\"name\":\"ambari-metrics-collector\"},' \ + ' {\"name\":\"hadooplzo_2_2_2_0_2538-native\"}]' + + self.executeScript("scripts/install_packages.py", + classname="InstallPackages", + command="actionexecute", + config_dict=config_dict, + target=RMFTestCase.TARGET_CUSTOM_ACTIONS, + os_type=('Suse', '11', 'Final'), + ) + self.assertTrue(put_structured_out_mock.called) + self.assertEquals(put_structured_out_mock.call_args[0][0], + {'package_installation_result': 'SUCCESS', + 'installed_repository_version': u'2.2.0.1-885', + 'stack_id': 'HDP-2.2', + 'ambari_repositories': []}) + self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885', + base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0', + action=['create'], + components=[u'HDP-UTILS', 'main'], + repo_template='repo_suse_rhel.j2', + repo_file_name=u'HDP-2.2.0.1-885', + mirror_list=None, + append_to_file=False, + ) + self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885', + base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0', + action=['create'], + components=[u'HDP', 'main'], + repo_template='repo_suse_rhel.j2', + repo_file_name=u'HDP-2.2.0.1-885', + mirror_list=None, + append_to_file=True, + ) + self.assertResourceCalled('Package', 'hive_2_2_0_1_885*', + use_repos=['base', 'HDP-UTILS-2.2.0.1-885', + 'HDP-2.2.0.1-885'], + ) + self.assertResourceCalled('Package', 'hive_2_2_0_1_885*-hcatalog', + use_repos=['base', 'HDP-UTILS-2.2.0.1-885', + 'HDP-2.2.0.1-885'], + ) + self.assertResourceCalled('Package', 'hive_2_2_0_1_885*-webhcat', + use_repos=['base', 'HDP-UTILS-2.2.0.1-885', + 'HDP-2.2.0.1-885'], + ) + self.assertResourceCalled('Package', 'hadooplzo_2_2_2_0_2538-native', + use_repos=['base', 'HDP-UTILS-2.2.0.1-885', + 'HDP-2.2.0.1-885'], + ) + self.assertNoMoreResources() + http://git-wip-us.apache.org/repos/asf/ambari/blob/8ce17b99/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json b/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json index 4f262ea..f9476a4 100644 --- a/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json +++ b/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json @@ -24,7 +24,7 @@ "roleParams": { "stack_id": "HDP-2.2", "repository_version": "2.2.0.1-885", - "base_urls": "[{\"name\":\"HDP-UTILS\",\"baseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-UTILS-1.1.0.20\"},{\"name\":\"HDP\",\"baseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-2.2\"}]";, + "base_urls": "[{\"name\":\"HDP-UTILS\",\"baseUrl\":\"http://repo1/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-UTILS-1.1.0.20\"},{\"name\":\"HDP\",\"baseUrl\":\"http://repo1/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-2.2\"}]";, "package_list": "[{\"name\":\"hadoop_2_2_*\"},{\"name\":\"snappy\"},{\"name\":\"snappy-devel\"},{\"name\":\"lzo\"},{\"name\":\"hadooplzo_2_2_*\"},{\"name\":\"hadoop_2_2_*-libhdfs\"},{\"name\":\"ambari-log4j\"}]" }, "serviceName": "null", @@ -37,7 +37,7 @@ "command_timeout": "60", "script_type": "PYTHON", "repository_version": "2.2.0.1-885", - "base_urls": "[{\"name\":\"HDP-UTILS\",\"baseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-UTILS-1.1.0.20\"},{\"name\":\"HDP\",\"baseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-2.2\"}]";, + "base_urls": "[{\"name\":\"HDP-UTILS\",\"baseUrl\":\"http://repo1/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-UTILS-1.1.0.20\"},{\"name\":\"HDP\",\"baseUrl\":\"http://repo1/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-2.2\"}]";, "package_list": "[{\"name\":\"hadoop_2_2_*\"},{\"name\":\"snappy\"},{\"name\":\"snappy-devel\"},{\"name\":\"lzo\"},{\"name\":\"hadooplzo_2_2_*\"},{\"name\":\"hadoop_2_2_*-libhdfs\"},{\"name\":\"ambari-log4j\"}]", "script": "install_packages.py" },
