AMBARI-21091. HDP deploy from public repo failed (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/871f85b3 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/871f85b3 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/871f85b3 Branch: refs/heads/branch-feature-AMBARI-20859 Commit: 871f85b34e5d0586dd21ed332a16abd012a72eef Parents: 7750c9a Author: Lisnichenko Dmitro <[email protected]> Authored: Mon Jun 12 19:38:34 2017 +0300 Committer: Lisnichenko Dmitro <[email protected]> Committed: Mon Jun 12 19:38:34 2017 +0300 ---------------------------------------------------------------------- .../test/python/ambari_agent/TestHostInfo.py | 107 ++++++++++ .../libraries/functions/packages_analyzer.py | 53 ++++- .../libraries/script/script.py | 25 ++- .../server/orm/entities/UpgradeEntity.java | 3 +- .../orm/entities/UpgradeHistoryEntity.java | 3 +- .../custom_actions/scripts/install_packages.py | 8 +- .../upgrades/UpgradeActionTest.java | 2 +- .../custom_actions/TestInstallPackages.py | 202 +++++++------------ .../configs/install_packages_config.json | 19 +- .../stacks/2.0.6/HBASE/test_hbase_master.py | 11 +- .../stacks/2.0.6/configs/hbase_with_phx.json | 12 ++ utility/pom.xml | 21 ++ 12 files changed, 322 insertions(+), 144 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/871f85b3/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py index a00771d..027a0a8 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py +++ b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py @@ -538,5 +538,112 @@ class TestHostInfo(TestCase): os_path_isfile_mock.return_value = False self.assertEqual("", hostInfo.getTransparentHugePage()) + @staticmethod + def _add_packages_available(command, arg): + arg.append(["hadoop_2_2_0_1_885", "1.0", "HDP-2.2"]) + arg.append(["hadooplzo_2_2_0_1_885", "1.0", "HDP-2.2"]) + arg.append(["hadoop_2_2_0_1_885-libhdfs", "1.0", "HDP-2.2"]) + + @staticmethod + def _add_packages_lookUpYum(command, key, arg): + TestHostInfo._add_packages_available(command, arg) + + @patch("ambari_commons.os_check.OSCheck.is_suse_family") + @patch("ambari_commons.os_check.OSCheck.is_ubuntu_family") + @patch("ambari_commons.os_check.OSCheck.is_redhat_family") + @patch("resource_management.libraries.functions.packages_analyzer._lookUpZypperPackages") + def test_get_available_packages_in_repos_suse(self, lookUpZypperPackages, is_redhat_family, is_ubuntu_family, + is_suse_family_mock): + is_suse_family_mock.return_value = True + is_redhat_family.return_value = False + is_ubuntu_family.return_value = False + lookUpZypperPackages.side_effect = TestHostInfo._add_packages_available + + command_json = { + "repositoryFile": { + "stackName": "HDP", + "repoVersionId": 1, + "repoVersion": "2", + "repositories": [ + { + "repoName": "HDP", + "baseUrl": "http://repo1/HDP/centos5/2.x/updates/2.2.0.0";, + "repoId": "HDP-2.2" + } + ] + } + } + + available_packages_in_repos = packages_analyzer.get_available_packages_in_repos( + command_json['repositoryFile']['repositories']) + + self.assertEqual(available_packages_in_repos, + ["hadoop_2_2_0_1_885", "hadooplzo_2_2_0_1_885", "hadoop_2_2_0_1_885-libhdfs"]) + + @patch("ambari_commons.os_check.OSCheck.is_suse_family") + @patch("ambari_commons.os_check.OSCheck.is_ubuntu_family") + @patch("ambari_commons.os_check.OSCheck.is_redhat_family") + @patch("resource_management.libraries.functions.packages_analyzer._lookUpYumPackages") + def test_get_available_packages_in_repos_rhel(self, lookUpYumPackages, is_redhat_family, is_ubuntu_family, + is_suse_family_mock): + is_suse_family_mock.return_value = False + is_redhat_family.return_value = True + is_ubuntu_family.return_value = False + lookUpYumPackages.side_effect = TestHostInfo._add_packages_lookUpYum + + command_json = { + "repositoryFile": { + "stackName": "HDP", + "repoVersionId": 1, + "repoVersion": "2", + "repositories": [ + { + "repoName": "HDP", + "baseUrl": "http://repo1/HDP/centos5/2.x/updates/2.2.0.0";, + "repoId": "HDP-2.2" + } + ] + } + } + + available_packages_in_repos = packages_analyzer.get_available_packages_in_repos( + command_json['repositoryFile']['repositories']) + + self.assertEqual(available_packages_in_repos, + ["hadoop_2_2_0_1_885", "hadooplzo_2_2_0_1_885", "hadoop_2_2_0_1_885-libhdfs", "hadoop_2_2_0_1_885", + "hadooplzo_2_2_0_1_885", "hadoop_2_2_0_1_885-libhdfs"]) + + @patch("ambari_commons.os_check.OSCheck.is_suse_family") + @patch("ambari_commons.os_check.OSCheck.is_ubuntu_family") + @patch("ambari_commons.os_check.OSCheck.is_redhat_family") + @patch("resource_management.libraries.functions.packages_analyzer._lookUpAptPackages") + def test_get_available_packages_in_repos_ubuntu(self, lookUpAptPackages, is_redhat_family, is_ubuntu_family, + is_suse_family_mock): + is_suse_family_mock.return_value = False + is_redhat_family.return_value = False + is_ubuntu_family.return_value = True + lookUpAptPackages.side_effect = TestHostInfo._add_packages_available + + command_json = { + "repositoryFile": { + "stackName": "HDP", + "repoVersionId": 1, + "repoVersion": "2", + "repositories": [ + { + "repoName": "HDP", + "baseUrl": "http://repo1/HDP/centos5/2.x/updates/2.2.0.0";, + "repoId": "HDP-2.2" + } + ] + } + } + + available_packages_in_repos = packages_analyzer.get_available_packages_in_repos( + command_json['repositoryFile']['repositories']) + + self.assertEqual(available_packages_in_repos, + ["hadoop_2_2_0_1_885", "hadooplzo_2_2_0_1_885", "hadoop_2_2_0_1_885-libhdfs"]) + if __name__ == "__main__": unittest.main() http://git-wip-us.apache.org/repos/asf/ambari/blob/871f85b3/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py b/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py index 8a15ee5..f4db3d2 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py @@ -34,8 +34,8 @@ __all__ = ["installedPkgsByName", "allInstalledPackages", "allAvailablePackages" "getInstalledRepos", "getInstalledPkgsByRepo", "getInstalledPkgsByNames", "getPackageDetails"] LIST_INSTALLED_PACKAGES_UBUNTU = "COLUMNS=9999 ; for i in $(dpkg -l |grep ^ii |awk -F' ' '{print $2}'); do apt-cache showpkg \"$i\"|head -3|grep -v '^Versions'| tr -d '()' | awk '{ print $1\" \"$2 }'|sed -e 's/^Package: //;' | paste -d ' ' - -; done" -LIST_AVAILABLE_PACKAGES_UBUNTU = "packages=`for i in $(ls -1 /var/lib/apt/lists | grep -v \"ubuntu.com\") ; do grep ^Package: /var/lib/apt/lists/$i | awk '{print $2}' ; done` ; for i in $packages; do apt-cache showpkg \"$i\"|head -3|grep -v '^Versions'| tr -d '()' | awk '{ print $1\" \"$2 }'|sed -e 's/^Package: //;' | paste -d ' ' - -; done" - +LIST_AVAILABLE_PACKAGES_UBUNTU = "packages=`for i in $(ls -1 /var/lib/apt/lists | grep %s ) ; do grep ^Package: /var/lib/apt/lists/$i | awk '{print $2}' ; done` ; for i in $packages; do apt-cache showpkg \"$i\"|head -3|grep -v '^Versions'| tr -d '()' | awk '{ print $1\" \"$2 }'|sed -e 's/^Package: //;' | paste -d ' ' - -; done" +GREP_REPO_EXCLUDE_SYSTEM = "-v \"ubuntu.com\"" logger = logging.getLogger() # default timeout for async invoked processes @@ -91,8 +91,6 @@ def allInstalledPackages(allInstalledPackages): """ All installed packages in system """ - osType = OSCheck.get_os_family() - if OSCheck.is_suse_family(): return _lookUpZypperPackages( ["sudo", "zypper", "--no-gpg-checks", "search", "--installed-only", "--details"], @@ -108,9 +106,47 @@ def allInstalledPackages(allInstalledPackages): allInstalledPackages) -def allAvailablePackages(allAvailablePackages): - osType = OSCheck.get_os_family() +def get_available_packages_in_repos(repositories): + """ + Gets all (both installed and available) packages that are available at given repositories. + :param repositories: from command configs like config['repositoryFile']['repositories'] + :return: installed and available packages from these repositories + """ + available_packages = [] + installed_packages = [] + available_packages_in_repos = [] + repo_ids = [repository['repoId'] for repository in repositories] + if OSCheck.is_ubuntu_family(): + allInstalledPackages(installed_packages) + repo_urls = [repository['baseUrl'] for repository in repositories] + repo_urls = [repo_url.replace("http://","";) for repo_url in repo_urls] + repo_urls = [repo_url.replace("/","_") for repo_url in repo_urls] + for url in repo_urls: + _lookUpAptPackages( + LIST_AVAILABLE_PACKAGES_UBUNTU % url, + available_packages) + for package in installed_packages: + if url in package[2]: + available_packages_in_repos.append(package[0]) + for package in available_packages: + available_packages_in_repos.append(package[0]) + elif OSCheck.is_suse_family(): + for repo in repo_ids: + _lookUpZypperPackages(["sudo", "zypper", "--no-gpg-checks", "search", "--details", "--repo", repo], + available_packages) + available_packages_in_repos += [package[0] for package in available_packages] + elif OSCheck.is_redhat_family(): + for repo in repo_ids: + _lookUpYumPackages(["sudo", "yum", "list", "available", "--disablerepo=*", "--enablerepo=" + repo], + 'Available Packages', available_packages) + _lookUpYumPackages(["sudo", "yum", "list", "installed", "--disablerepo=*", "--enablerepo=" + repo], + 'Installed Packages', installed_packages) + available_packages_in_repos += [package[0] for package in available_packages + installed_packages] + return available_packages_in_repos + + +def allAvailablePackages(allAvailablePackages): if OSCheck.is_suse_family(): return _lookUpZypperPackages( ["sudo", "zypper", "--no-gpg-checks", "search", "--uninstalled-only", "--details"], @@ -122,7 +158,7 @@ def allAvailablePackages(allAvailablePackages): allAvailablePackages) elif OSCheck.is_ubuntu_family(): return _lookUpAptPackages( - LIST_AVAILABLE_PACKAGES_UBUNTU, + LIST_AVAILABLE_PACKAGES_UBUNTU % GREP_REPO_EXCLUDE_SYSTEM, allAvailablePackages) # ToDo: add execution via sudo for ubuntu (currently Ubuntu is not supported) @@ -170,7 +206,6 @@ def _lookUpZypperPackages(command, allPackages): if 0 == result['retCode']: lines = result['out'].split('\n') lines = [line.strip() for line in lines] - items = [] for index in range(len(lines)): if "--+--" in lines[index]: skipIndex = index + 1 @@ -281,7 +316,7 @@ def getInstalledPackageVersion(package_name): code, out, err = rmf_shell.checked_call("dpkg -s {0} | grep Version | awk '{{print $2}}'".format(package_name), stderr=subprocess.PIPE) else: code, out, err = rmf_shell.checked_call("rpm -q --queryformat '%{{version}}-%{{release}}' {0} | sed -e 's/\.el[0-9]//g'".format(package_name), stderr=subprocess.PIPE) - + return out http://git-wip-us.apache.org/repos/asf/ambari/blob/871f85b3/ambari-common/src/main/python/resource_management/libraries/script/script.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index 054490b..2c56a13 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -463,10 +463,26 @@ class Script(object): return Script.stack_version_from_distro_select + + def get_package_from_available(self, name, available_packages_in_repos): + """ + This function matches package names with ${stack_version} placeholder to actual package names from + Ambari-managed repository. + Package names without ${stack_version} placeholder are returned as is. + """ + if STACK_VERSION_PLACEHOLDER not in name: + return name + package_delimiter = '-' if OSCheck.is_ubuntu_family() else '_' + package_regex = name.replace(STACK_VERSION_PLACEHOLDER, '(\d|{0})+'.format(package_delimiter)) + for package in available_packages_in_repos: + if re.match(package_regex, package): + return package + + def format_package_name(self, name): from resource_management.libraries.functions.default import default """ - This function replaces ${stack_version} placeholder into actual version. If the package + This function replaces ${stack_version} placeholder with actual version. If the package version is passed from the server, use that as an absolute truth. """ @@ -703,12 +719,15 @@ class Script(object): package_list_str = config['hostLevelParams']['package_list'] agent_stack_retry_on_unavailability = bool(config['hostLevelParams']['agent_stack_retry_on_unavailability']) agent_stack_retry_count = int(config['hostLevelParams']['agent_stack_retry_count']) - + try: + available_packages_in_repos = packages_analyzer.get_available_packages_in_repos(config['repositoryFile']['repositories']) + except Exception as err: + available_packages_in_repos = [] if isinstance(package_list_str, basestring) and len(package_list_str) > 0: package_list = json.loads(package_list_str) for package in package_list: if self.check_package_condition(package): - name = self.format_package_name(package['name']) + name = self.get_package_from_available(package['name'], available_packages_in_repos) # HACK: On Windows, only install ambari-metrics packages using Choco Package Installer # TODO: Update this once choco packages for hadoop are created. This is because, service metainfo.xml support # <osFamily>any<osFamily> which would cause installation failure on Windows. http://git-wip-us.apache.org/repos/asf/ambari/blob/871f85b3/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/UpgradeEntity.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/UpgradeEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/UpgradeEntity.java index 975d6a6..43b2e08 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/UpgradeEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/UpgradeEntity.java @@ -41,7 +41,8 @@ import org.apache.ambari.server.actionmanager.HostRoleStatus; import org.apache.ambari.server.state.stack.upgrade.Direction; import org.apache.ambari.server.state.stack.upgrade.UpgradeType; import org.apache.commons.lang.builder.EqualsBuilder; -import org.apache.hadoop.metrics2.sink.relocated.google.common.base.Objects; + +import com.google.common.base.Objects; /** * Models the data representation of an upgrade http://git-wip-us.apache.org/repos/asf/ambari/blob/871f85b3/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/UpgradeHistoryEntity.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/UpgradeHistoryEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/UpgradeHistoryEntity.java index 8bfafd3..0f7ac72 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/UpgradeHistoryEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/UpgradeHistoryEntity.java @@ -31,7 +31,8 @@ import javax.persistence.TableGenerator; import javax.persistence.UniqueConstraint; import org.apache.commons.lang.builder.EqualsBuilder; -import org.apache.hadoop.metrics2.sink.relocated.google.common.base.Objects; + +import com.google.common.base.Objects; /** * The {@link UpgradeHistoryEntity} represents the version history of components http://git-wip-us.apache.org/repos/asf/ambari/blob/871f85b3/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py index d6da0fc..a4a7a8c 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py @@ -22,6 +22,7 @@ import signal import re import ambari_simplejson as json +import sys, traceback from ambari_commons.os_check import OSCheck from ambari_commons.str_utils import cbool, cint @@ -36,6 +37,7 @@ from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions.repo_version_history \ import read_actual_version_from_history_file, write_actual_version_to_history_file, REPO_VERSION_HISTORY_FILE from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions import packages_analyzer from resource_management.libraries.functions.repository_util import create_repo_files, CommandRepository from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.resources.repository import Repository @@ -352,8 +354,12 @@ class InstallPackages(Script): packages_installed_before = [package[0] for package in packages_installed_before] packages_were_checked = True filtered_package_list = self.filter_package_list(package_list) + try: + available_packages_in_repos = packages_analyzer.get_available_packages_in_repos(config['repositoryFile']['repositories']) + except Exception: + available_packages_in_repos = [] for package in filtered_package_list: - name = self.format_package_name(package['name']) + name = self.get_package_from_available(package['name'], available_packages_in_repos) Package(name, action="upgrade", # this enables upgrading non-versioned packages, despite the fact they exist. Needed by 'mahout' which is non-version but have to be updated retry_on_repo_unavailability=agent_stack_retry_on_unavailability, http://git-wip-us.apache.org/repos/asf/ambari/blob/871f85b3/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java index e3ffae0..50d219c 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java @@ -79,7 +79,7 @@ import org.apache.ambari.server.state.UpgradeState; import org.apache.ambari.server.state.stack.UpgradePack; import org.apache.ambari.server.state.stack.upgrade.UpgradeType; import org.apache.ambari.server.utils.EventBusSynchronizer; -import org.apache.hadoop.metrics2.sink.relocated.commons.lang.StringUtils; +import org.apache.commons.lang.StringUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; http://git-wip-us.apache.org/repos/asf/ambari/blob/871f85b3/ambari-server/src/test/python/custom_actions/TestInstallPackages.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/custom_actions/TestInstallPackages.py b/ambari-server/src/test/python/custom_actions/TestInstallPackages.py index 5f5576e..3e90c64 100644 --- a/ambari-server/src/test/python/custom_actions/TestInstallPackages.py +++ b/ambari-server/src/test/python/custom_actions/TestInstallPackages.py @@ -61,8 +61,18 @@ class TestInstallPackages(RMFTestCase): arg.append(["pkg1", "1.0", "repo"]) arg.append(["pkg2", "2.0", "repo2"]) + @staticmethod + def _add_packages_available(command, arg): + arg.append(["hadoop_2_2_0_1_885", "1.0", "HDP-2.2"]) + arg.append(["hadooplzo_2_2_0_1_885", "1.0", "HDP-2.2"]) + arg.append(["hadoop_2_2_0_1_885-libhdfs", "1.0", "HDP-2.2"]) + + @staticmethod + def _add_packages_lookUpYum(command, key, arg): + TestInstallPackages._add_packages_available(command, arg) + @patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos") - @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages") + @patch("resource_management.libraries.functions.packages_analyzer._lookUpYumPackages") @patch("resource_management.libraries.script.Script.put_structured_out") @patch("resource_management.libraries.functions.stack_select.get_stack_versions") @patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file") @@ -71,12 +81,13 @@ class TestInstallPackages(RMFTestCase): write_actual_version_to_history_file_mock, read_actual_version_from_history_file_mock, stack_versions_mock, - put_structured_out_mock, allInstalledPackages_mock, list_ambari_managed_repos_mock): + put_structured_out_mock, + lookUpYumPackages, list_ambari_managed_repos_mock): stack_versions_mock.side_effect = [ [], # before installation attempt [VERSION_STUB] ] - allInstalledPackages_mock.side_effect = TestInstallPackages._add_packages + lookUpYumPackages.side_effect = TestInstallPackages._add_packages_lookUpYum list_ambari_managed_repos_mock.return_value=[] self.executeScript("scripts/install_packages.py", classname="InstallPackages", @@ -91,21 +102,21 @@ class TestInstallPackages(RMFTestCase): 'installed_repository_version': VERSION_STUB, 'stack_id': 'HDP-2.2', 'actual_version': VERSION_STUB}) - self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885', + self.assertResourceCalled('Repository', 'HDP-UTILS-1.1.0.20', base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0', action=['create'], components=[u'HDP-UTILS', 'main'], repo_template='[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0', - repo_file_name=u'HDP-2.2.0.1-885', + repo_file_name=u'ambari-hdp-1', mirror_list=None, append_to_file=False, ) - self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885', + self.assertResourceCalled('Repository', 'HDP-2.2', base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0', action=['create'], components=[u'HDP', 'main'], repo_template='[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0', - repo_file_name=u'HDP-2.2.0.1-885', + repo_file_name=u'ambari-hdp-1', mirror_list=None, append_to_file=True, ) @@ -140,6 +151,7 @@ class TestInstallPackages(RMFTestCase): command_json = json.load(f) command_json['roleParams']['base_urls'] = "[]" + del command_json['repositoryFile'] allInstalledPackages_mock.side_effect = TestInstallPackages._add_packages @@ -159,32 +171,33 @@ class TestInstallPackages(RMFTestCase): 'actual_version': VERSION_STUB}) self.assertResourceCalled('Package', 'hdp-select', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) - self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', None, action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) self.assertResourceCalled('Package', 'snappy', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) self.assertResourceCalled('Package', 'snappy-devel', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) self.assertResourceCalled('Package', 'lzo', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) - self.assertResourceCalled('Package', 'hadooplzo_2_2_0_1_885', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) - self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885-libhdfs', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', None, action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', None, action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) self.assertResourceCalled('Package', 'ambari-log4j', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) self.assertNoMoreResources() @patch("ambari_commons.os_check.OSCheck.is_suse_family") @patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos") - @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages") + @patch("resource_management.libraries.functions.packages_analyzer._lookUpZypperPackages") @patch("resource_management.libraries.script.Script.put_structured_out") @patch("resource_management.libraries.functions.stack_select.get_stack_versions") @patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file") @patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file") def test_normal_flow_sles(self, write_actual_version_to_history_file_mock, read_actual_version_from_history_file_mock, - stack_versions_mock, put_structured_out_mock, allInstalledPackages_mock, list_ambari_managed_repos_mock, is_suse_family_mock): + stack_versions_mock, put_structured_out_mock, + lookUpZypperPackages, list_ambari_managed_repos_mock, is_suse_family_mock): is_suse_family_mock = True Script.stack_version_from_distro_select = VERSION_STUB stack_versions_mock.side_effect = [ [], # before installation attempt [VERSION_STUB] ] - allInstalledPackages_mock.side_effect = TestInstallPackages._add_packages + lookUpZypperPackages.side_effect = TestInstallPackages._add_packages_available list_ambari_managed_repos_mock.return_value=[] self.executeScript("scripts/install_packages.py", classname="InstallPackages", @@ -199,21 +212,21 @@ class TestInstallPackages(RMFTestCase): 'installed_repository_version': VERSION_STUB, 'stack_id': 'HDP-2.2', 'actual_version': VERSION_STUB}) - self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885', + self.assertResourceCalled('Repository', 'HDP-UTILS-1.1.0.20', base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0', action=['create'], components=[u'HDP-UTILS', 'main'], repo_template='[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0', - repo_file_name=u'HDP-2.2.0.1-885', + repo_file_name=u'ambari-hdp-1', mirror_list=None, append_to_file=False, ) - self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885', + self.assertResourceCalled('Repository', 'HDP-2.2', base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0', action=['create'], components=[u'HDP', 'main'], repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0', - repo_file_name=u'HDP-2.2.0.1-885', + repo_file_name=u'ambari-hdp-1', mirror_list=None, append_to_file=True, ) @@ -231,21 +244,21 @@ class TestInstallPackages(RMFTestCase): @patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos") @patch("ambari_commons.os_check.OSCheck.is_redhat_family") @patch("resource_management.libraries.script.Script.put_structured_out") - @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages") + @patch("resource_management.libraries.functions.packages_analyzer._lookUpYumPackages") @patch("resource_management.libraries.functions.stack_select.get_stack_versions") @patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file") @patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file") - def test_exclude_existing_repo(self, write_actual_version_to_history_file_mock, + def test_exclude_existing_repo(self, write_actual_version_to_history_file_mock, read_actual_version_from_history_file_mock, stack_versions_mock, - allInstalledPackages_mock, put_structured_out_mock, + lookUpYumPackages, put_structured_out_mock, is_redhat_family_mock, list_ambari_managed_repos_mock): stack_versions_mock.side_effect = [ [], # before installation attempt [VERSION_STUB] ] Script.stack_version_from_distro_select = VERSION_STUB - allInstalledPackages_mock.side_effect = TestInstallPackages._add_packages + lookUpYumPackages.side_effect = TestInstallPackages._add_packages_lookUpYum list_ambari_managed_repos_mock.return_value=["HDP-UTILS-2.2.0.1-885"] is_redhat_family_mock.return_value = True self.executeScript("scripts/install_packages.py", @@ -261,21 +274,21 @@ class TestInstallPackages(RMFTestCase): 'installed_repository_version': VERSION_STUB, 'stack_id': 'HDP-2.2', 'actual_version': VERSION_STUB}) - self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885', + self.assertResourceCalled('Repository', 'HDP-UTILS-1.1.0.20', base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0', action=['create'], components=[u'HDP-UTILS', 'main'], repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0', - repo_file_name='HDP-2.2.0.1-885', + repo_file_name='ambari-hdp-1', mirror_list=None, append_to_file=False, ) - self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885', + self.assertResourceCalled('Repository', 'HDP-2.2', base_url='http://repo1/HDP/centos5/2.x/updates/2.2.0.0', action=['create'], components=[u'HDP', 'main'], repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0', - repo_file_name=u'HDP-2.2.0.1-885', + repo_file_name=u'ambari-hdp-1', mirror_list=None, append_to_file=True, ) @@ -337,21 +350,21 @@ class TestInstallPackages(RMFTestCase): {'stack_id': 'HDP-2.2', 'installed_repository_version': VERSION_STUB, 'package_installation_result': 'FAIL'}) - self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885', + self.assertResourceCalled('Repository', 'HDP-UTILS-1.1.0.20', base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0', action=['create'], components=[u'HDP-UTILS', 'main'], repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0', - repo_file_name=u'HDP-2.2.0.1-885', + repo_file_name=u'ambari-hdp-1', mirror_list=None, append_to_file=False, ) - self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885', + self.assertResourceCalled('Repository', 'HDP-2.2', base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0', action=['create'], components=[u'HDP', 'main'], repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0', - repo_file_name=u'HDP-2.2.0.1-885', + repo_file_name=u'ambari-hdp-1', mirror_list=None, append_to_file=True, ) @@ -363,14 +376,14 @@ class TestInstallPackages(RMFTestCase): @patch("ambari_commons.os_check.OSCheck.is_suse_family") @patch("resource_management.core.resources.packaging.Package") @patch("resource_management.libraries.script.Script.put_structured_out") - @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages") + @patch("resource_management.libraries.functions.packages_analyzer._lookUpZypperPackages") @patch("resource_management.libraries.functions.stack_select.get_stack_versions") @patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file") @patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file") - def test_format_package_name(self, write_actual_version_to_history_file_mock, + def test_format_package_name(self, write_actual_version_to_history_file_mock, read_actual_version_from_history_file_mock, stack_versions_mock, - allInstalledPackages_mock, put_structured_out_mock, + lookUpZypperPackages, put_structured_out_mock, package_mock, is_suse_family_mock): Script.stack_version_from_distro_select = VERSION_STUB stack_versions_mock.side_effect = [ @@ -378,7 +391,7 @@ class TestInstallPackages(RMFTestCase): [VERSION_STUB] ] read_actual_version_from_history_file_mock.return_value = VERSION_STUB - allInstalledPackages_mock = MagicMock(side_effect = TestInstallPackages._add_packages) + lookUpZypperPackages.side_effect = TestInstallPackages._add_packages_available is_suse_family_mock.return_value = True self.executeScript("scripts/install_packages.py", classname="InstallPackages", @@ -393,21 +406,21 @@ class TestInstallPackages(RMFTestCase): 'installed_repository_version': VERSION_STUB, 'stack_id': 'HDP-2.2', 'actual_version': VERSION_STUB}) - self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885', + self.assertResourceCalled('Repository', 'HDP-UTILS-1.1.0.20', base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0', action=['create'], components=[u'HDP-UTILS', 'main'], repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0', - repo_file_name=u'HDP-2.2.0.1-885', + repo_file_name=u'ambari-hdp-1', mirror_list=None, append_to_file=False, ) - self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885', + self.assertResourceCalled('Repository', 'HDP-2.2', base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0', action=['create'], components=[u'HDP', 'main'], repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0', - repo_file_name=u'HDP-2.2.0.1-885', + repo_file_name=u'ambari-hdp-1', mirror_list=None, append_to_file=True, ) @@ -421,99 +434,30 @@ class TestInstallPackages(RMFTestCase): self.assertResourceCalled('Package', 'ambari-log4j', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) self.assertNoMoreResources() - @patch("ambari_commons.os_check.OSCheck.is_suse_family") - @patch("resource_management.core.resources.packaging.Package") - @patch("resource_management.libraries.script.Script.put_structured_out") - @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages") - @patch("resource_management.libraries.functions.stack_select.get_stack_versions") - @patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file") - @patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file") - def test_format_package_name_specific(self, write_actual_version_to_history_file_mock, - read_actual_version_from_history_file_mock, - stack_versions_mock, - allInstalledPackages_mock, put_structured_out_mock, - package_mock, is_suse_family_mock): - Script.stack_version_from_distro_select = VERSION_STUB - stack_versions_mock.side_effect = [ - [], # before installation attempt - [VERSION_STUB] - ] - read_actual_version_from_history_file_mock.return_value = VERSION_STUB - allInstalledPackages_mock = MagicMock(side_effect = TestInstallPackages._add_packages) - is_suse_family_mock.return_value = True - - - config_file = self.get_src_folder() + "/test/python/custom_actions/configs/install_packages_config.json" - with open(config_file, "r") as f: - command_json = json.load(f) - - command_json['roleParams']['package_version'] = "2_2_0_1_889" - - - self.executeScript("scripts/install_packages.py", - classname="InstallPackages", - command="actionexecute", - config_dict=command_json, - target=RMFTestCase.TARGET_CUSTOM_ACTIONS, - os_type=('Suse', '11', 'Final'), - ) - self.assertTrue(put_structured_out_mock.called) - self.assertEquals(put_structured_out_mock.call_args[0][0], - {'package_installation_result': 'SUCCESS', - 'installed_repository_version': VERSION_STUB, - 'stack_id': 'HDP-2.2', - 'actual_version': VERSION_STUB}) - self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885', - base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0', - action=['create'], - components=[u'HDP-UTILS', 'main'], - repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0', - repo_file_name=u'HDP-2.2.0.1-885', - mirror_list=None, - append_to_file=False, - ) - self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885', - base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0', - action=['create'], - components=[u'HDP', 'main'], - repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0', - repo_file_name=u'HDP-2.2.0.1-885', - mirror_list=None, - append_to_file=True, - ) - self.assertResourceCalled('Package', 'hdp-select', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) - self.assertResourceCalled('Package', 'hadoop_2_2_0_1_889', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) - self.assertResourceCalled('Package', 'snappy', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) - self.assertResourceCalled('Package', 'snappy-devel', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) - self.assertResourceCalled('Package', 'lzo', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) - self.assertResourceCalled('Package', 'hadooplzo_2_2_0_1_889', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) - self.assertResourceCalled('Package', 'hadoop_2_2_0_1_889-libhdfs', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) - self.assertResourceCalled('Package', 'ambari-log4j', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) - self.assertNoMoreResources() @patch("ambari_commons.os_check.OSCheck.is_suse_family") @patch("resource_management.core.resources.packaging.Package") @patch("resource_management.libraries.script.Script.put_structured_out") - @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages") + @patch("resource_management.libraries.functions.packages_analyzer._lookUpZypperPackages") @patch("resource_management.libraries.functions.stack_select.get_stack_versions") @patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file") @patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file") def test_format_package_name_via_repositoryFile(self, write_actual_version_to_history_file_mock, - read_actual_version_from_history_file_mock, - stack_versions_mock, - allInstalledPackages_mock, put_structured_out_mock, - package_mock, is_suse_family_mock): + read_actual_version_from_history_file_mock, + stack_versions_mock, + lookUpZypperPackages, put_structured_out_mock, + package_mock, is_suse_family_mock): Script.stack_version_from_distro_select = VERSION_STUB stack_versions_mock.side_effect = [ [], # before installation attempt [VERSION_STUB] ] read_actual_version_from_history_file_mock.return_value = VERSION_STUB - allInstalledPackages_mock = MagicMock(side_effect = TestInstallPackages._add_packages) + lookUpZypperPackages.side_effect = TestInstallPackages._add_packages_available is_suse_family_mock.return_value = True - + config_file = self.get_src_folder() + "/test/python/custom_actions/configs/install_packages_repository_file.json" with open(config_file, "r") as f: command_json = json.load(f) @@ -552,12 +496,15 @@ class TestInstallPackages(RMFTestCase): append_to_file=True, ) self.assertResourceCalled('Package', 'hdp-select', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) - self.assertResourceCalled('Package', 'hadoop_2_2_0_1_990', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885', action=["upgrade"], retry_count=5, + retry_on_repo_unavailability=False) self.assertResourceCalled('Package', 'snappy', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) self.assertResourceCalled('Package', 'snappy-devel', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) self.assertResourceCalled('Package', 'lzo', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) - self.assertResourceCalled('Package', 'hadooplzo_2_2_0_1_990', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) - self.assertResourceCalled('Package', 'hadoop_2_2_0_1_990-libhdfs', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadooplzo_2_2_0_1_885', action=["upgrade"], retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885-libhdfs', action=["upgrade"], retry_count=5, + retry_on_repo_unavailability=False) self.assertResourceCalled('Package', 'ambari-log4j', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) self.assertNoMoreResources() @@ -1106,21 +1053,23 @@ class TestInstallPackages(RMFTestCase): self.assertFalse(write_actual_version_to_history_file_mock.called) @patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos") - @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages") + @patch("resource_management.libraries.functions.packages_analyzer._lookUpYumPackages") @patch("resource_management.libraries.script.Script.put_structured_out") @patch("resource_management.libraries.functions.stack_select.get_stack_versions") @patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file") @patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file") def test_normal_flow_rhel_with_command_repo(self, - write_actual_version_to_history_file_mock, - read_actual_version_from_history_file_mock, - stack_versions_mock, - put_structured_out_mock, allInstalledPackages_mock, list_ambari_managed_repos_mock): + write_actual_version_to_history_file_mock, + read_actual_version_from_history_file_mock, + stack_versions_mock, + put_structured_out_mock, + lookUpYumPackages, + list_ambari_managed_repos_mock): stack_versions_mock.side_effect = [ [], # before installation attempt [VERSION_STUB] ] - allInstalledPackages_mock.side_effect = TestInstallPackages._add_packages + lookUpYumPackages.side_effect = TestInstallPackages._add_packages_lookUpYum list_ambari_managed_repos_mock.return_value=[] self.executeScript("scripts/install_packages.py", classname="InstallPackages", @@ -1155,11 +1104,14 @@ class TestInstallPackages(RMFTestCase): append_to_file=True, ) self.assertResourceCalled('Package', 'hdp-select', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) - self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885', action=["upgrade"], retry_count=5, + retry_on_repo_unavailability=False) self.assertResourceCalled('Package', 'snappy', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) self.assertResourceCalled('Package', 'snappy-devel', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) self.assertResourceCalled('Package', 'lzo', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) - self.assertResourceCalled('Package', 'hadooplzo_2_2_0_1_885', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) - self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885-libhdfs', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadooplzo_2_2_0_1_885', action=["upgrade"], retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885-libhdfs', action=["upgrade"], retry_count=5, + retry_on_repo_unavailability=False) self.assertResourceCalled('Package', 'ambari-log4j', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) self.assertNoMoreResources() http://git-wip-us.apache.org/repos/asf/ambari/blob/871f85b3/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json b/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json index c6575b5..be010b3 100644 --- a/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json +++ b/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json @@ -49,7 +49,24 @@ "package_list": "[{\"name\":\"hadoop_${stack_version}\", \"condition\": \"\"},{\"name\":\"snappy\", \"condition\": \"\"},{\"name\":\"snappy-devel\", \"condition\": \"\"},{\"name\":\"lzo\", \"condition\": \"\"},{\"name\":\"hadooplzo_${stack_version}\", \"condition\": \"\"},{\"name\":\"hadoop_${stack_version}-libhdfs\", \"condition\": \"\"},{\"name\":\"ambari-log4j\", \"condition\": \"\"}]", "script": "install_packages.py" }, - "commandId": "14-1", + "commandId": "14-1", + "repositoryFile": { + "stackName": "HDP", + "repoVersionId": 1, + "repoVersion": "2", + "repositories": [ + { + "repoName": "HDP-UTILS", + "baseUrl": "http://repo1/HDP/centos5/2.x/updates/2.2.0.0";, + "repoId": "HDP-UTILS-1.1.0.20" + }, + { + "repoName": "HDP", + "baseUrl": "http://repo1/HDP/centos5/2.x/updates/2.2.0.0";, + "repoId": "HDP-2.2" + } + ] + }, "clusterHostInfo": { "snamenode_host": [ "0b3.vm" http://git-wip-us.apache.org/repos/asf/ambari/blob/871f85b3/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py index fda63e0..05fa204 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py @@ -69,11 +69,17 @@ class TestHBaseMaster(RMFTestCase): self.assertNoMoreResources() - def test_install_hbase_master_with_version(self): + @patch("resource_management.libraries.functions.packages_analyzer._lookUpYumPackages") + def test_install_hbase_master_with_version(self, lookUpYumPackages): + def _add_packages_available(command, key, arg): + arg.append(["hbase_2_3_0_1_1234", "1.0", "testrepo"]) + config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/hbase_with_phx.json" with open(config_file, "r") as f: json_content = json.load(f) version = '2.3.0.1-1234' + + lookUpYumPackages.side_effect = _add_packages_available # the json file is not a "well formed" install command json_content['roleCommand'] = 'INSTALL' json_content['commandParams']['version'] = version @@ -86,8 +92,9 @@ class TestHBaseMaster(RMFTestCase): stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES, try_install=True, + os_type=('Redhat', '6.4', 'Final'), checked_call_mocks = [(0, "OK.", "")], - ) + ) # only assert that the correct package is trying to be installed self.assertResourceCalled('Package', 'hbase_2_3_0_1_1234', http://git-wip-us.apache.org/repos/asf/ambari/blob/871f85b3/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_with_phx.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_with_phx.json b/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_with_phx.json index 5147603..9979e9d 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_with_phx.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_with_phx.json @@ -28,6 +28,18 @@ "roleParams": {}, "serviceName": "HIVE", "role": "HIVE_SERVER", + "repositoryFile": { + "stackName": "HDP", + "repoVersionId": 1, + "repoVersion": "2", + "repositories": [ + { + "repoName": "HDP", + "baseUrl": "testurl", + "repoId": "testrepo" + } + ] + }, "commandParams": { "command_timeout": "300", "service_package_folder": "OOZIE", http://git-wip-us.apache.org/repos/asf/ambari/blob/871f85b3/utility/pom.xml ---------------------------------------------------------------------- diff --git a/utility/pom.xml b/utility/pom.xml index 7d5eb93..918080e 100644 --- a/utility/pom.xml +++ b/utility/pom.xml @@ -67,6 +67,27 @@ </configuration> </plugin> <plugin> + <groupId>org.vafer</groupId> + <artifactId>jdeb</artifactId> + <version>1.0.1</version> + <executions> + <execution> + <!--Stub execution on direct plugin call - workaround for ambari deb build process--> + <id>stub-execution</id> + <phase>none</phase> + <goals> + <goal>jdeb</goal> + </goals> + </execution> + </executions> + <configuration> + <skip>true</skip> + <attach>false</attach> + <submodules>false</submodules> + <controlDir>${project.basedir}/../src/main/package/deb/control</controlDir> + </configuration> + </plugin> + <plugin> <artifactId>maven-assembly-plugin</artifactId> <configuration> <descriptors>
