Repository: ambari Updated Branches: refs/heads/branch-feature-AMBARI-22457 c7457a820 -> aea1a3c48
AMBARI-22489. Port AMBARI-22467 to BigInsight (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/aea1a3c4 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/aea1a3c4 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/aea1a3c4 Branch: refs/heads/branch-feature-AMBARI-22457 Commit: aea1a3c48fdf58be5fad04d7b983c65d28a248f9 Parents: c7457a8 Author: Andrew Onishuk <[email protected]> Authored: Wed Nov 22 12:32:04 2017 +0200 Committer: Andrew Onishuk <[email protected]> Committed: Wed Nov 22 12:32:04 2017 +0200 ---------------------------------------------------------------------- .../BigInsights/4.0/services/HDFS/metainfo.xml | 17 ------------ .../BigInsights/4.2/services/HDFS/metainfo.xml | 20 -------------- .../4.2/services/HDFS/package/scripts/hdfs.py | 5 ++-- .../4.2/services/HIVE/package/scripts/hive.py | 3 +++ .../4.2/services/OOZIE/package/scripts/oozie.py | 12 +++++---- .../services/OOZIE/package/scripts/params.py | 28 ++------------------ .../4.2/services/YARN/package/scripts/yarn.py | 3 +++ 7 files changed, 17 insertions(+), 71 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml index 918cdb3..82c82d0 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml @@ -152,10 +152,6 @@ <package> <name>hadoop</name> </package> - <package> - <name>hadoop-lzo</name> - <condition>should_install_lzo</condition> - </package> </packages> </osSpecific> @@ -165,15 +161,6 @@ <package> <name>snappy</name> </package> - <package> - <name>lzo</name> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadoop-lzo-native</name> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop-libhdfs</name> </package> <package> @@ -192,10 +179,6 @@ <name>libsnappy-dev</name> </package> <package> - <name>liblzo2-2</name> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop-hdfs</name> </package> <package> http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml index 72ce2be..4253eb1 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml @@ -182,11 +182,6 @@ <package> <name>hadoop_4_2_*</name> </package> - <package> - <name>hadoop-lzo</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> </packages> </osSpecific> <osSpecific> @@ -196,16 +191,6 @@ <name>snappy</name> </package> <package> - <name>lzo</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadoop-lzo-native</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop_4_2_*-libhdfs</name> </package> <package> @@ -229,11 +214,6 @@ <name>libsnappy-dev</name> </package> <package> - <name>liblzo2-2</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop-hdfs</name> </package> <package> http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py index 2b158bd..9334de0 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py @@ -20,10 +20,10 @@ Ambari Agent """ from resource_management import * +from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed import sys import os - def hdfs(name=None): import params @@ -115,8 +115,7 @@ def hdfs(name=None): content=Template("slaves.j2") ) - if params.lzo_enabled and len(params.lzo_packages) > 0: - Package(params.lzo_packages) + install_lzo_if_needed() def install_snappy(): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py index 0446e1c..f451a65 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py @@ -34,6 +34,7 @@ from resource_management.core.shell import as_user from resource_management.libraries.functions.is_empty import is_empty from resource_management.libraries.resources.xml_config import XmlConfig from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed from resource_management.core.exceptions import Fail from resource_management.core.shell import as_sudo from resource_management.core.shell import quote_bash_args @@ -48,6 +49,8 @@ from ambari_commons import OSConst def hive(name=None): import params + install_lzo_if_needed() + XmlConfig("hive-site.xml", conf_dir = params.hive_conf_dir, configurations = params.config['configurations']['hive-site'], http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py index 64a83c0..0459432 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py @@ -26,6 +26,7 @@ from resource_management.core.resources.system import Execute from resource_management.core.source import DownloadSource from resource_management.core.source import InlineTemplate from resource_management.core.source import Template +from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed from resource_management.libraries.functions import format from resource_management.libraries.functions import compare_versions from resource_management.libraries.resources.xml_config import XmlConfig @@ -99,6 +100,12 @@ def oozie(is_server=False): pass oozie_ownership() + + if params.lzo_enabled: + install_lzo_if_needed() + Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), + not_if = no_op_test, + ) if is_server: oozie_server_specific() @@ -227,11 +234,6 @@ def oozie_server_specific(): Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension.jar'), not_if = no_op_test, ) - #if params.lzo_enabled and len(params.lzo_packages_for_current_host) > 0: - # Package(params.lzo_packages_for_current_host) - # Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), - # not_if = no_op_test, - # ) prepare_war_cmd_file = format("{oozie_home}/.prepare_war_cmd") prepare_war_cmd = format("cd {oozie_tmp_dir} && {oozie_setup_sh} prepare-war {oozie_secure}") http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py index d1a4a66..ee9b88a 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py @@ -27,6 +27,7 @@ from resource_management.libraries.functions import get_port_from_url from resource_management.libraries.functions import format from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions.version import compare_versions +from resource_management.libraries.functions.lzo_utils import should_install_lzo from resource_management.libraries.resources import HdfsResource from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select @@ -234,29 +235,4 @@ HdfsResource = functools.partial( default_fs = default_fs ) -#LZO support - -#-----LZO is not suppported in IOP distribution since it is GPL license-------- - -''' -io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) -lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() - -# stack_is_iop40_or_further -underscored_version = stack_version_unformatted.replace('.', '_') -dashed_version = stack_version_unformatted.replace('.', '-') -lzo_packages_to_family = { - "any": ["hadoop-lzo", ], - "redhat": ["lzo", "hadoop-lzo-native"], - "suse": ["lzo", "hadoop-lzo-native"], - "ubuntu": ["liblzo2-2", ] -} - - -lzo_packages_to_family["redhat"] += [format("hadooplzo_{underscorred_version}_*")] -lzo_packages_to_family["suse"] += [format("hadooplzo_{underscorred_version}_*")] -lzo_packages_to_family["ubuntu"] += [format("hadooplzo_{dashed_version}_*")] - -lzo_packages_for_current_host = lzo_packages_to_family['any'] + lzo_packages_to_family[System.get_instance().os_family] -all_lzo_packages = set(itertools.chain(*lzo_packages_to_family.values())) -''' +lzo_enabled = should_install_lzo() http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py index a284183..9006d2e 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py @@ -20,6 +20,7 @@ Ambari Agent """ from resource_management import * +from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed import sys import os @@ -50,6 +51,8 @@ def create_local_dir(dir_name): def yarn(name = None): import params + + install_lzo_if_needed() if name == "historyserver": if params.yarn_log_aggregation_enabled:
