AMBARI-14739. Capture package requirements that are optional and config dependent (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9f76f601 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9f76f601 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9f76f601 Branch: refs/heads/branch-dev-patch-upgrade Commit: 9f76f6019d09464e0f3fd04aba3b02bf059d003d Parents: 8b7aa92 Author: Andrew Onishuk <aonis...@hortonworks.com> Authored: Wed Jan 20 19:08:04 2016 +0200 Committer: Andrew Onishuk <aonis...@hortonworks.com> Committed: Wed Jan 20 19:08:04 2016 +0200 ---------------------------------------------------------------------- .../python/resource_management/TestScript.py | 4 +- .../libraries/functions/get_lzo_packages.py | 5 +- .../libraries/functions/package_conditions.py | 64 ++++++++++++++++++++ .../libraries/script/script.py | 21 ++++++- .../2.1.1/services/AMBARI_METRICS/metainfo.xml | 1 + .../ambari/server/state/ServiceOsSpecific.java | 17 +++++- .../0.1.0/package/scripts/metrics_monitor.py | 4 +- .../0.96.0.2.0/package/scripts/hbase_client.py | 4 +- .../0.96.0.2.0/package/scripts/hbase_master.py | 3 +- .../package/scripts/hbase_regionserver.py | 3 +- .../0.96.0.2.0/package/scripts/params_linux.py | 5 -- .../package/scripts/phoenix_queryserver.py | 3 +- .../common-services/HDFS/2.1.0.2.0/metainfo.xml | 4 ++ .../HDFS/2.1.0.2.0/package/scripts/datanode.py | 4 +- .../2.1.0.2.0/package/scripts/hdfs_client.py | 4 +- .../2.1.0.2.0/package/scripts/journalnode.py | 4 +- .../HDFS/2.1.0.2.0/package/scripts/namenode.py | 4 +- .../2.1.0.2.0/package/scripts/nfsgateway.py | 2 +- .../2.1.0.2.0/package/scripts/params_linux.py | 4 -- .../HDFS/2.1.0.2.0/package/scripts/snamenode.py | 2 +- .../2.1.0.2.0/package/scripts/zkfc_slave.py | 2 +- .../HIVE/0.12.0.2.0/metainfo.xml | 4 ++ .../0.12.0.2.0/package/scripts/hcat_client.py | 2 +- .../0.12.0.2.0/package/scripts/hive_client.py | 2 +- .../package/scripts/hive_metastore.py | 2 +- .../0.12.0.2.0/package/scripts/hive_server.py | 2 +- .../0.12.0.2.0/package/scripts/mysql_server.py | 2 +- .../0.12.0.2.0/package/scripts/params_linux.py | 18 +----- .../package/scripts/params_windows.py | 2 - .../package/scripts/webhcat_server.py | 2 +- .../KERBEROS/1.10.3-10/metainfo.xml | 6 ++ .../package/scripts/kerberos_client.py | 2 +- .../SQOOP/1.4.4.2.0/metainfo.xml | 1 + .../custom_actions/scripts/install_packages.py | 17 ++---- .../stacks/HDP/2.1/services/HIVE/metainfo.xml | 4 ++ .../stacks/HDP/2.2/services/HBASE/metainfo.xml | 2 + .../stacks/HDP/2.2/services/HDFS/metainfo.xml | 4 ++ .../stacks/HDP/2.2/services/HIVE/metainfo.xml | 5 ++ .../stacks/HDP/2.2/services/OOZIE/metainfo.xml | 1 + .../stacks/HDP/2.2/services/SQOOP/metainfo.xml | 1 + .../stacks/HDP/2.3/services/HDFS/metainfo.xml | 3 + .../stacks/HDP/2.3/services/HIVE/metainfo.xml | 7 +++ .../stacks/HDP/2.3/services/OOZIE/metainfo.xml | 1 + .../configs/install_packages_config.json | 4 +- .../stacks/2.0.6/configs/hbase_no_phx.json | 2 +- .../stacks/2.0.6/configs/hbase_with_phx.json | 2 +- .../python/stacks/2.3/configs/pxf_default.json | 2 +- 47 files changed, 184 insertions(+), 80 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-agent/src/test/python/resource_management/TestScript.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestScript.py b/ambari-agent/src/test/python/resource_management/TestScript.py index fd31910..f6a5c8c 100644 --- a/ambari-agent/src/test/python/resource_management/TestScript.py +++ b/ambari-agent/src/test/python/resource_management/TestScript.py @@ -59,8 +59,8 @@ class TestScript(TestCase): } dummy_config = { 'hostLevelParams' : { - 'package_list' : "[{\"type\":\"rpm\",\"name\":\"hbase\"}," - "{\"type\":\"rpm\",\"name\":\"yet-another-package\"}]", + 'package_list' : "[{\"type\":\"rpm\",\"name\":\"hbase\", \"condition\": \"\"}," + "{\"type\":\"rpm\",\"name\":\"yet-another-package\", \"condition\": \"\"}]", 'repo_info' : "[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]";, 'service_repo_info' : "[{\"mirrorsList\":\"abc\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]"; } http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py index 0537ec3..afb4314 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py @@ -38,10 +38,7 @@ def get_lzo_packages(stack_version_unformatted): hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0: - if OSCheck.is_redhat_family() or OSCheck.is_suse_family(): - lzo_packages += [format("hadooplzo_*")] - elif OSCheck.is_ubuntu_family(): - lzo_packages += [format("hadooplzo_*")] + lzo_packages += ["hadooplzo_*"] else: lzo_packages += ["hadoop-lzo"] http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py new file mode 100644 index 0000000..0fb0fe4 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +__all__ = ["is_lzo_enabled", "should_install_phoenix", "should_install_ams_collector", "should_install_mysql", "should_install_mysl_connector"] + +import os +from resource_management.libraries.script import Script +from resource_management.libraries.functions.default import default + +def should_install_lzo(): + config = Script.get_config() + io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) + lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() + return lzo_enabled + +def should_install_phoenix(): + phoenix_hosts = default('/clusterHostInfo/phoenix_query_server_hosts', []) + phoenix_enabled = default('/configurations/hbase-env/phoenix_sql_enabled', False) + has_phoenix = len(phoenix_hosts) > 0 + return phoenix_enabled or has_phoenix + +def should_install_ams_collector(): + config = Script.get_config() + return 'role' in config and config['role'] == "METRICS_COLLECTOR" + +def should_install_mysql(): + config = Script.get_config() + hive_database = config['configurations']['hive-env']['hive_database'] + hive_use_existing_db = hive_database.startswith('Existing') + + if hive_use_existing_db or 'role' in config and config['role'] != "MYSQL_SERVER": + return False + return True + +def should_install_mysl_connector(): + mysql_jdbc_driver_jar = "/usr/share/java/mysql-connector-java.jar" + return not os.path.exists(mysql_jdbc_driver_jar) + +def should_install_hive_atlas(): + atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', []) + has_atlas = len(atlas_hosts) > 0 + return has_atlas + +def should_install_kerberos_server(): + config = Script.get_config() + return 'role' in config and config['role'] != "KERBEROS_CLIENT" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-common/src/main/python/resource_management/libraries/script/script.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index 7101386..3bef342 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -28,6 +28,7 @@ import logging import platform import inspect import tarfile +import resource_management from ambari_commons import OSCheck, OSConst from ambari_commons.constants import UPGRADE_TYPE_NON_ROLLING, UPGRADE_TYPE_ROLLING from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl @@ -420,7 +421,7 @@ class Script(object): """ self.install_packages(env) - def install_packages(self, env, exclude_packages=[]): + def install_packages(self, env): """ List of packages that are required< by service is received from the server as a command parameter. The method installs all packages @@ -442,7 +443,7 @@ class Script(object): if isinstance(package_list_str, basestring) and len(package_list_str) > 0: package_list = json.loads(package_list_str) for package in package_list: - if not Script.matches_any_regexp(package['name'], exclude_packages): + if Script.check_package_condition(package): name = self.format_package_name(package['name']) # HACK: On Windows, only install ambari-metrics packages using Choco Package Installer # TODO: Update this once choco packages for hadoop are created. This is because, service metainfo.xml support @@ -465,6 +466,22 @@ class Script(object): reload_windows_env() @staticmethod + def check_package_condition(package): + from resource_management.libraries.functions import package_conditions + condition = package['condition'] + name = package['name'] + + if not condition: + return True + + try: + chooser_method = getattr(package_conditions, condition) + except AttributeError: + raise Fail("Condition with name '{0}', when installing package {1}. Please check package_conditions.py.".format(condition, name)) + + return chooser_method() + + @staticmethod def matches_any_regexp(string, regexp_list): for regex in regexp_list: # we cannot use here Python regex, since * will create some troubles matching plaintext names. http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/services/AMBARI_METRICS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/services/AMBARI_METRICS/metainfo.xml b/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/services/AMBARI_METRICS/metainfo.xml index aea1252..84735c2 100644 --- a/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/services/AMBARI_METRICS/metainfo.xml +++ b/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/services/AMBARI_METRICS/metainfo.xml @@ -69,6 +69,7 @@ <packages> <package> <name>ambari-metrics-collector</name> + <condition>package_chooser.should_install_ams_collector()</condition> </package> <package> <name>ambari-metrics-monitor</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceOsSpecific.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceOsSpecific.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceOsSpecific.java index 47e244f..5e1d5d3 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceOsSpecific.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceOsSpecific.java @@ -165,6 +165,7 @@ public class ServiceOsSpecific { @XmlAccessorType(XmlAccessType.FIELD) public static class Package { private String name; + private String condition = ""; /** * If true, package will not be attempted to be upgraded during RU. @@ -180,6 +181,14 @@ public class ServiceOsSpecific { public void setName(String name) { this.name = name; } + + public String getCondition() { + return condition; + } + + public void setCondition(String condition) { + this.condition = condition; + } public Boolean getSkipUpgrade() { return skipUpgrade; @@ -200,14 +209,18 @@ public class ServiceOsSpecific { Package aPackage = (Package) o; if (!name.equals(aPackage.name)) return false; - return skipUpgrade.equals(aPackage.skipUpgrade); - + if (!skipUpgrade.equals(aPackage.skipUpgrade)) return false; + if (!condition.equals(aPackage.condition)) return false; + + return true; } @Override public int hashCode() { int result = name.hashCode(); result = 31 * result + skipUpgrade.hashCode(); + result = 31 * result + condition.hashCode(); + return result; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_monitor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_monitor.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_monitor.py index a1d6a1d..d98384b 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_monitor.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_monitor.py @@ -25,7 +25,9 @@ from status import check_service_status class AmsMonitor(Script): def install(self, env): - self.install_packages(env, exclude_packages = ['ambari-metrics-collector']) + import params + env.set_params(params) + self.install_packages(env) self.configure(env) # for security def configure(self, env): http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_client.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_client.py index 97213fd..ac34d40 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_client.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_client.py @@ -30,8 +30,8 @@ from ambari_commons.os_family_impl import OsFamilyImpl class HbaseClient(Script): def install(self, env): import params - - self.install_packages(env, params.exclude_packages) + env.set_params(params) + self.install_packages(env) self.configure(env) def configure(self, env): http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py index fdd8f5a..9515f61 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py @@ -40,7 +40,8 @@ class HbaseMaster(Script): def install(self, env): import params - self.install_packages(env, params.exclude_packages) + env.set_params(params) + self.install_packages(env) def decommission(self, env): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py index ecab844..c17d219 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py @@ -34,7 +34,8 @@ from ambari_commons.os_family_impl import OsFamilyImpl class HbaseRegionServer(Script): def install(self, env): import params - self.install_packages(env, params.exclude_packages) + env.set_params(params) + self.install_packages(env) def configure(self, env): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py index 4a44676..337f09d 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py @@ -113,11 +113,6 @@ phoenix_hosts = default('/clusterHostInfo/phoenix_query_server_hosts', []) phoenix_enabled = default('/configurations/hbase-env/phoenix_sql_enabled', False) has_phoenix = len(phoenix_hosts) > 0 -if not has_phoenix and not phoenix_enabled: - exclude_packages = ['phoenix*'] -else: - exclude_packages = [] - underscored_version = stack_version_unformatted.replace('.', '_') dashed_version = stack_version_unformatted.replace('.', '-') if OSCheck.is_redhat_family() or OSCheck.is_suse_family(): http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_queryserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_queryserver.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_queryserver.py index e894f95..693bb08 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_queryserver.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_queryserver.py @@ -28,7 +28,8 @@ class PhoenixQueryServer(Script): def install(self, env): import params - self.install_packages(env, params.exclude_packages) + env.set_params(params) + self.install_packages(env) def get_stack_to_component(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml index e3e16e3..752de0e 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml @@ -165,6 +165,7 @@ <package> <name>hadoop-lzo</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_lzo</condition> </package> </packages> </osSpecific> @@ -181,10 +182,12 @@ <package> <name>lzo</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_lzo</condition> </package> <package> <name>hadoop-lzo-native</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_lzo</condition> </package> <package> <name>hadoop-libhdfs</name> @@ -204,6 +207,7 @@ <package> <name>liblzo2-2</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_lzo</condition> </package> <package> <name>hadoop-hdfs</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py index 1d242e1..5adeab4 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py @@ -46,8 +46,8 @@ class DataNode(Script): def install(self, env): import params - self.install_packages(env, params.exclude_packages) env.set_params(params) + self.install_packages(env) def configure(self, env): import params @@ -161,7 +161,7 @@ class DataNodeDefault(DataNode): class DataNodeWindows(DataNode): def install(self, env): import install_params - self.install_packages(env, install_params.exclude_packages) + self.install_packages(env) if __name__ == "__main__": DataNode().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py index 16218b6..21c0eda 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py @@ -31,8 +31,8 @@ class HdfsClient(Script): def install(self, env): import params - self.install_packages(env, params.exclude_packages) env.set_params(params) + self.install_packages(env) self.configure(env) def configure(self, env): @@ -113,7 +113,7 @@ class HdfsClientDefault(HdfsClient): class HdfsClientWindows(HdfsClient): def install(self, env): import install_params - self.install_packages(env, install_params.exclude_packages) + self.install_packages(env) self.configure(env) if __name__ == "__main__": http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py index ad4283c..7715f6c 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py @@ -36,8 +36,8 @@ from ambari_commons import OSConst class JournalNode(Script): def install(self, env): import params - self.install_packages(env, params.exclude_packages) env.set_params(params) + self.install_packages(env) @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) class JournalNodeDefault(JournalNode): @@ -164,7 +164,7 @@ class JournalNodeDefault(JournalNode): class JournalNodeWindows(JournalNode): def install(self, env): import install_params - self.install_packages(env, install_params.exclude_packages) + self.install_packages(env) def start(self, env): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py index 9a42702..910bc0a 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py @@ -82,8 +82,8 @@ class NameNode(Script): def install(self, env): import params - self.install_packages(env, params.exclude_packages) env.set_params(params) + self.install_packages(env) #TODO we need this for HA because of manual steps self.configure(env) @@ -347,7 +347,7 @@ class NameNodeDefault(NameNode): class NameNodeWindows(NameNode): def install(self, env): import install_params - self.install_packages(env, install_params.exclude_packages) + self.install_packages(env) #TODO we need this for HA because of manual steps self.configure(env) http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/nfsgateway.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/nfsgateway.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/nfsgateway.py index df5569e..4b9ad06 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/nfsgateway.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/nfsgateway.py @@ -39,7 +39,7 @@ class NFSGateway(Script): env.set_params(params) - self.install_packages(env, params.exclude_packages) + self.install_packages(env) def pre_upgrade_restart(self, env, upgrade_type=None): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py index c923e23..29c4784 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py @@ -341,10 +341,6 @@ HdfsResource = functools.partial( io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() lzo_packages = get_lzo_packages(stack_version_unformatted) - -exclude_packages = [] -if not lzo_enabled: - exclude_packages += lzo_packages name_node_params = default("/commandParams/namenode", None) http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py index 1748955..b8a1726 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py @@ -36,7 +36,7 @@ class SNameNode(Script): def install(self, env): import params env.set_params(params) - self.install_packages(env, params.exclude_packages) + self.install_packages(env) def configure(self, env): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py index 148e605..3558681 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py @@ -39,7 +39,7 @@ class ZkfcSlave(Script): def install(self, env): import params env.set_params(params) - self.install_packages(env, params.exclude_packages) + self.install_packages(env) def configure(self, env): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/metainfo.xml index 634b2fc..fc93ee6 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/metainfo.xml @@ -255,6 +255,7 @@ <package> <name>mysql-connector-java</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_mysl_connector</condition> </package> </packages> </osSpecific> @@ -273,6 +274,7 @@ <package> <name>mysql-server</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_mysql</condition> </package> </packages> </osSpecific> @@ -282,10 +284,12 @@ <package> <name>mysql-community-release</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_mysql</condition> </package> <package> <name>mysql-community-server</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_mysql</condition> </package> </packages> </osSpecific> http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat_client.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat_client.py index b372085..70bebb4 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat_client.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat_client.py @@ -31,7 +31,7 @@ from resource_management.libraries.script.script import Script class HCatClient(Script): def install(self, env): import params - self.install_packages(env, exclude_packages=params.hive_exclude_packages) + self.install_packages(env) self.configure(env) def configure(self, env): http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_client.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_client.py index 117b456..e4aace3 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_client.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_client.py @@ -28,7 +28,7 @@ from ambari_commons import OSConst class HiveClient(Script): def install(self, env): import params - self.install_packages(env, exclude_packages=params.hive_exclude_packages) + self.install_packages(env) self.configure(env) def status(self, env): http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py index 5866cd0..59ecbbb 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py @@ -47,7 +47,7 @@ LEGACY_HIVE_SERVER_CONF = "/etc/hive/conf.server" class HiveMetastore(Script): def install(self, env): import params - self.install_packages(env, exclude_packages = params.hive_exclude_packages) + self.install_packages(env) def start(self, env, upgrade_type=None): http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py index 96c05ca..a81e4f6 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py @@ -47,7 +47,7 @@ from hive_service import hive_service class HiveServer(Script): def install(self, env): import params - self.install_packages(env, exclude_packages=params.hive_exclude_packages) + self.install_packages(env) def configure(self, env): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/mysql_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/mysql_server.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/mysql_server.py index dcfb567..851dc02 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/mysql_server.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/mysql_server.py @@ -30,7 +30,7 @@ from mysql_utils import mysql_configure class MysqlServer(Script): def install(self, env): import params - self.install_packages(env, exclude_packages=params.hive_exclude_packages) + self.install_packages(env) self.configure(env) def clean(self, env): http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index a2131b0..cd3741d 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -384,19 +384,6 @@ hive_authorization_enabled = config['configurations']['hive-site']['hive.securit mysql_jdbc_driver_jar = "/usr/share/java/mysql-connector-java.jar" hive_use_existing_db = hive_database.startswith('Existing') -hive_exclude_packages = [] - -# There are other packages that contain /usr/share/java/mysql-connector-java.jar (like libmysql-java), -# trying to install mysql-connector-java upon them can cause packages to conflict. -if hive_use_existing_db: - hive_exclude_packages = ['mysql-connector-java', 'mysql', 'mysql-server', - 'mysql-community-release', 'mysql-community-server'] -else: - if 'role' in config and config['role'] != "MYSQL_SERVER": - hive_exclude_packages = ['mysql', 'mysql-server', 'mysql-community-release', - 'mysql-community-server'] - if os.path.exists(mysql_jdbc_driver_jar): - hive_exclude_packages.append('mysql-connector-java') hive_site_config = dict(config['configurations']['hive-site']) @@ -410,10 +397,7 @@ classpath_addition = "" atlas_plugin_package = "atlas-metadata*-hive-plugin" atlas_ubuntu_plugin_package = "atlas-metadata.*-hive-plugin" -if not has_atlas: - hive_exclude_packages.append(atlas_plugin_package) - hive_exclude_packages.append(atlas_ubuntu_plugin_package) -else: +if has_atlas: # client.properties atlas_client_props = {} auth_enabled = config['configurations']['application-properties'].get( http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_windows.py index f2524b2..359604f 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_windows.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_windows.py @@ -61,8 +61,6 @@ hive_metastore_db_type = config['configurations']['hive-env']['hive_database_typ hive_metastore_user_name = config['configurations']['hive-site']['javax.jdo.option.ConnectionUserName'] hive_metastore_user_passwd = config['configurations']['hive-site']['javax.jdo.option.ConnectionPassword'] -hive_exclude_packages = [] - hive_execution_engine = config["configurations"]["hive-site"]["hive.execution.engine"] ######## Metastore Schema http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py index f0f31a8..c12b168 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py @@ -33,7 +33,7 @@ from ambari_commons.os_family_impl import OsFamilyImpl class WebHCatServer(Script): def install(self, env): import params - self.install_packages(env, exclude_packages=params.hive_exclude_packages) + self.install_packages(env) def start(self, env, upgrade_type=None): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/KERBEROS/1.10.3-10/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KERBEROS/1.10.3-10/metainfo.xml b/ambari-server/src/main/resources/common-services/KERBEROS/1.10.3-10/metainfo.xml index 72f159a..ec462a7 100644 --- a/ambari-server/src/main/resources/common-services/KERBEROS/1.10.3-10/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/KERBEROS/1.10.3-10/metainfo.xml @@ -78,10 +78,12 @@ <package> <name>krb5-server</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_kerberos_server</condition> </package> <package> <name>krb5-libs</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_kerberos_server</condition> </package> <package> <name>krb5-workstation</name> @@ -96,10 +98,12 @@ <package> <name>krb5-kdc</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_kerberos_server</condition> </package> <package> <name>krb5-admin-server</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_kerberos_server</condition> </package> <package> <name>krb5-user</name> @@ -118,6 +122,7 @@ <package> <name>krb5</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_kerberos_server</condition> </package> <package> <name>krb5-client</name> @@ -126,6 +131,7 @@ <package> <name>krb5-server</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_kerberos_server</condition> </package> </packages> </osSpecific> http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/KERBEROS/1.10.3-10/package/scripts/kerberos_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KERBEROS/1.10.3-10/package/scripts/kerberos_client.py b/ambari-server/src/main/resources/common-services/KERBEROS/1.10.3-10/package/scripts/kerberos_client.py index 580545d..013aabc 100644 --- a/ambari-server/src/main/resources/common-services/KERBEROS/1.10.3-10/package/scripts/kerberos_client.py +++ b/ambari-server/src/main/resources/common-services/KERBEROS/1.10.3-10/package/scripts/kerberos_client.py @@ -24,7 +24,7 @@ class KerberosClient(KerberosScript): def install(self, env): install_packages = default('/configurations/kerberos-env/install_packages', "true") if install_packages: - self.install_packages(env, ['krb5-server', 'krb5-libs', 'krb5-auth-dialog', 'krb5', 'krb5-kdc', 'krb5-admin-server']) + self.install_packages(env) else: print "Kerberos client packages are not being installed, manual installation is required." http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml index 0cf9c7d..b5db91b 100644 --- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml @@ -72,6 +72,7 @@ <package> <name>mysql-connector-java</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_mysl_connector</condition> </package> </packages> </osSpecific> http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py index b28a5e0..bf80616 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py @@ -28,6 +28,7 @@ import os.path import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. from resource_management import * +import resource_management from resource_management.libraries.functions.list_ambari_managed_repos import list_ambari_managed_repos from ambari_commons.os_check import OSCheck, OSConst from resource_management.libraries.functions.packages_analyzer import allInstalledPackages @@ -416,25 +417,15 @@ class InstallPackages(Script): def filter_package_list(self, package_list): """ Note: that we have skipUpgrade option in metainfo.xml to filter packages, + as well as condition option to filter them conditionally, so use this method only if, for some reason the metainfo option cannot be used. - - Here we filter packages that are managed with custom logic in package - scripts. Usually this packages come from system repositories, and either - are not available when we restrict repository list, or should not be - installed on host at all. + :param package_list: original list :return: filtered package_list """ filtered_package_list = [] for package in package_list: - skip_package = False - - # skip upgrade for hadooplzo* versioned package, only if lzo is disabled - io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) - if not io_compression_codecs or "com.hadoop.compression.lzo" not in io_compression_codecs: - skip_package = package['name'].startswith('hadooplzo') - - if not skip_package: + if Script.check_package_condition(package): filtered_package_list.append(package) return filtered_package_list http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/metainfo.xml index 5c9e8c2..279bcb7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/metainfo.xml @@ -58,6 +58,7 @@ <package> <name>mysql-connector-java</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_mysl_connector</condition> </package> </packages> </osSpecific> @@ -76,10 +77,12 @@ <package> <name>mysql-community-release</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_mysql</condition> </package> <package> <name>mysql-community-server</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_mysql</condition> </package> </packages> </osSpecific> @@ -89,6 +92,7 @@ <package> <name>mysql-server</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_mysql</condition> </package> </packages> </osSpecific> http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/metainfo.xml index d80b5d1..e88f073 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/metainfo.xml @@ -32,6 +32,7 @@ </package> <package> <name>phoenix_${stack_version}</name> + <condition>should_install_phoenix</condition> </package> </packages> </osSpecific> @@ -43,6 +44,7 @@ </package> <package> <name>phoenix-${stack_version}</name> + <condition>should_install_phoenix</condition> </package> </packages> </osSpecific> http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml index ca4b5c5..0c98a72 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml @@ -39,9 +39,11 @@ <package> <name>lzo</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_lzo</condition> </package> <package> <name>hadooplzo_${stack_version}</name> + <condition>should_install_lzo</condition> </package> <package> <name>hadoop_${stack_version}-libhdfs</name> @@ -78,10 +80,12 @@ </package> <package> <name>hadooplzo-${stack_version}</name> + <condition>should_install_lzo</condition> </package> <package> <name>liblzo2-2</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_lzo</condition> </package> <package> <name>libhdfs0-${stack_version}</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/metainfo.xml index ba87d8e..11921a6 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/metainfo.xml @@ -63,6 +63,7 @@ <package> <name>mysql-connector-java</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_mysl_connector</condition> </package> </packages> </osSpecific> @@ -95,10 +96,12 @@ <package> <name>mysql-community-release</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_mysql</condition> </package> <package> <name>mysql-community-server</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_mysql</condition> </package> </packages> </osSpecific> @@ -122,6 +125,7 @@ <package> <name>mysql-server</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_mysql</condition> </package> </packages> </osSpecific> @@ -131,6 +135,7 @@ <package> <name>mysql-client</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_mysql</condition> </package> </packages> </osSpecific> http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/stacks/HDP/2.2/services/OOZIE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/OOZIE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/OOZIE/metainfo.xml index d5db93c..b5682e4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/OOZIE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/OOZIE/metainfo.xml @@ -67,6 +67,7 @@ <package> <name>mysql-connector-java</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_mysl_connector</condition> </package> <package> <name>extjs</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/stacks/HDP/2.2/services/SQOOP/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/SQOOP/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/SQOOP/metainfo.xml index eaa4051..d408f38 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/SQOOP/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/SQOOP/metainfo.xml @@ -28,6 +28,7 @@ <package> <name>mysql-connector-java</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_mysl_connector</condition> </package> </packages> </osSpecific> http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml index cb0062f..ac6ddaf 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml @@ -70,9 +70,11 @@ <package> <name>lzo</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_lzo</condition> </package> <package> <name>hadooplzo_${stack_version}</name> + <condition>should_install_lzo</condition> </package> <package> <name>hadoop_${stack_version}-libhdfs</name> @@ -109,6 +111,7 @@ </package> <package> <name>hadooplzo-${stack_version}</name> + <condition>should_install_lzo</condition> </package> <package> <name>libhdfs0-${stack_version}</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/metainfo.xml index f184741..fcbb02e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/metainfo.xml @@ -29,6 +29,7 @@ <package> <name>mysql-connector-java</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_mysl_connector</condition> </package> </packages> </osSpecific> @@ -46,6 +47,7 @@ </package> <package> <name>atlas-metadata_${stack_version}-hive-plugin</name> + <condition>should_install_hive_atlas</condition> </package> </packages> </osSpecific> @@ -55,6 +57,7 @@ <package> <name>mysql</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_mysql</condition> </package> </packages> </osSpecific> @@ -64,10 +67,12 @@ <package> <name>mysql-community-release</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_mysql</condition> </package> <package> <name>mysql-community-server</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_mysql</condition> </package> </packages> </osSpecific> @@ -85,6 +90,7 @@ </package> <package> <name>atlas-metadata-${stack_version}-hive-plugin</name> + <condition>should_install_hive_atlas</condition> </package> </packages> </osSpecific> @@ -94,6 +100,7 @@ <package> <name>mysql-server</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_mysql</condition> </package> </packages> </osSpecific> http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/main/resources/stacks/HDP/2.3/services/OOZIE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/OOZIE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/OOZIE/metainfo.xml index 8d1dda1..9b324bb 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/OOZIE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/OOZIE/metainfo.xml @@ -38,6 +38,7 @@ <package> <name>mysql-connector-java</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_mysl_connector</condition> </package> <package> <name>extjs</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json b/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json index fcd7765..957a3d8 100644 --- a/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json +++ b/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json @@ -26,7 +26,7 @@ "stack_id": "HDP-2.2", "repository_version": "2.2.0.1-885", "base_urls": "[{\"name\":\"HDP-UTILS\",\"baseUrl\":\"http://repo1/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-UTILS-1.1.0.20\"},{\"name\":\"HDP\",\"baseUrl\":\"http://repo1/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-2.2\"}]";, - "package_list": "[{\"name\":\"hadoop_${stack_version}\"},{\"name\":\"snappy\"},{\"name\":\"snappy-devel\"},{\"name\":\"lzo\"},{\"name\":\"hadooplzo_${stack_version}\"},{\"name\":\"hadoop_${stack_version}-libhdfs\"},{\"name\":\"ambari-log4j\"}]" + "package_list": "[{\"name\":\"hadoop_${stack_version}\", \"condition\": \"\"},{\"name\":\"snappy\", \"condition\": \"\"},{\"name\":\"snappy-devel\", \"condition\": \"\"},{\"name\":\"lzo\", \"condition\": \"\"},{\"name\":\"hadooplzo_${stack_version}\", \"condition\": \"\"},{\"name\":\"hadoop_${stack_version}-libhdfs\", \"condition\": \"\"},{\"name\":\"ambari-log4j\", \"condition\": \"\"}]" }, "serviceName": "null", "role": "install_packages", @@ -44,7 +44,7 @@ "script_type": "PYTHON", "repository_version": "2.2.0.1-885", "base_urls": "[{\"name\":\"HDP-UTILS\",\"baseUrl\":\"http://repo1/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-UTILS-1.1.0.20\"},{\"name\":\"HDP\",\"baseUrl\":\"http://repo1/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-2.2\"}]";, - "package_list": "[{\"name\":\"hadoop_${stack_version}\"},{\"name\":\"snappy\"},{\"name\":\"snappy-devel\"},{\"name\":\"lzo\"},{\"name\":\"hadooplzo_${stack_version}\"},{\"name\":\"hadoop_${stack_version}-libhdfs\"},{\"name\":\"ambari-log4j\"}]", + "package_list": "[{\"name\":\"hadoop_${stack_version}\", \"condition\": \"\"},{\"name\":\"snappy\", \"condition\": \"\"},{\"name\":\"snappy-devel\", \"condition\": \"\"},{\"name\":\"lzo\", \"condition\": \"\"},{\"name\":\"hadooplzo_${stack_version}\", \"condition\": \"\"},{\"name\":\"hadoop_${stack_version}-libhdfs\", \"condition\": \"\"},{\"name\":\"ambari-log4j\", \"condition\": \"\"}]", "script": "install_packages.py" }, "commandId": "14-1", http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_no_phx.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_no_phx.json b/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_no_phx.json index e2e9965..1c2f04d 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_no_phx.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_no_phx.json @@ -16,7 +16,7 @@ "java_home": "/usr/jdk64/jdk1.7.0_45", "java_version": "8", "host_sys_prepped": "false", - "package_list": "[{\"name\":\"hbase_2_3_*\",\"skipUpgrade\":false},{\"name\":\"phoenix_2_3_*\",\"skipUpgrade\":false}]", + "package_list": "[{\"name\":\"hbase_2_3_*\",\"condition\":\"\",\"skipUpgrade\":false},{\"name\":\"phoenix_*\",\"condition\":\"{{has_phoenix or phoenix_enabled}}\",\"skipUpgrade\":false}]", "db_name": "ambari", "group_list": "[\"hadoop\",\"nobody\",\"users\"]", "user_list": "[\"hive\",\"oozie\",\"nobody\",\"ambari-qa\",\"flume\",\"hdfs\",\"storm\",\"mapred\",\"hbase\",\"tez\",\"zookeeper\",\"falcon\",\"sqoop\",\"yarn\",\"hcat\"]" http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_with_phx.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_with_phx.json b/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_with_phx.json index 1a3b217..78131b1 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_with_phx.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_with_phx.json @@ -16,7 +16,7 @@ "java_home": "/usr/jdk64/jdk1.7.0_45", "java_version": "8", "host_sys_prepped": "false", - "package_list": "[{\"name\":\"hbase_2_3_*\",\"skipUpgrade\":false},{\"name\":\"phoenix_2_3_*\",\"skipUpgrade\":false}]", + "package_list": "[{\"name\":\"hbase_2_3_*\",\"condition\":\"\",\"skipUpgrade\":false},{\"name\":\"phoenix_2_3_*\",\"condition\":\"{{has_phoenix or phoenix_enabled}}\",\"skipUpgrade\":false}]", "db_name": "ambari", "group_list": "[\"hadoop\",\"nobody\",\"users\"]", "user_list": "[\"hive\",\"oozie\",\"nobody\",\"ambari-qa\",\"flume\",\"hdfs\",\"storm\",\"mapred\",\"hbase\",\"tez\",\"zookeeper\",\"falcon\",\"sqoop\",\"yarn\",\"hcat\"]" http://git-wip-us.apache.org/repos/asf/ambari/blob/9f76f601/ambari-server/src/test/python/stacks/2.3/configs/pxf_default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/configs/pxf_default.json b/ambari-server/src/test/python/stacks/2.3/configs/pxf_default.json index a042dfd..ba06162 100644 --- a/ambari-server/src/test/python/stacks/2.3/configs/pxf_default.json +++ b/ambari-server/src/test/python/stacks/2.3/configs/pxf_default.json @@ -14,7 +14,7 @@ "ambari_db_rca_username": "mapred", "java_home": "/usr/jdk64/jdk1.7.0_45", "db_name": "ambari", - "package_list": "[{\"name\":\"pxf-service\",\"skipUpgrade\":false},{\"name\":\"apache-tomcat\",\"skipUpgrade\":false},{\"name\":\"pxf-hive\",\"skipUpgrade\":false},{\"name\":\"pxf-hdfs\",\"skipUpgrade\":false},{\"name\":\"pxf-hbase\",\"skipUpgrade\":false}]" + "package_list": "[{\"name\":\"pxf-service\",\"skipUpgrade\":false, \"condition\": \"\"},{\"name\":\"apache-tomcat\",\"skipUpgrade\":false, \"condition\": \"\"},{\"name\":\"pxf-hive\",\"skipUpgrade\":false, \"condition\": \"\"},{\"name\":\"pxf-hdfs\",\"skipUpgrade\":false, \"condition\": \"\"},{\"name\":\"pxf-hbase\",\"skipUpgrade\":false, \"condition\": \"\"}]" }, "commandType": "EXECUTION_COMMAND", "roleParams": {},
