Repository: ambari Updated Branches: refs/heads/branch-2.6 88f0dc30f -> 6df336124
AMBARI-22383. Remove Auto-Installation of LZO Libraries (ncole) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6df33612 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6df33612 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6df33612 Branch: refs/heads/branch-2.6 Commit: 6df336124e3ad13c0793bd32b35931ded5219f2d Parents: 88f0dc3 Author: Nate Cole <[email protected]> Authored: Wed Nov 8 10:45:13 2017 -0500 Committer: Nate Cole <[email protected]> Committed: Wed Nov 8 12:04:11 2017 -0500 ---------------------------------------------------------------------- .../libraries/functions/__init__.py | 1 - .../libraries/functions/get_lzo_packages.py | 50 -------------------- .../libraries/functions/package_conditions.py | 8 +--- .../common-services/HDFS/2.1.0.2.0/metainfo.xml | 30 ------------ .../HDFS/2.1.0.2.0/package/scripts/datanode.py | 1 - .../HDFS/2.1.0.2.0/package/scripts/hdfs.py | 5 -- .../2.1.0.2.0/package/scripts/hdfs_client.py | 1 - .../2.1.0.2.0/package/scripts/install_params.py | 11 +---- .../2.1.0.2.0/package/scripts/journalnode.py | 1 - .../HDFS/2.1.0.2.0/package/scripts/namenode.py | 1 - .../2.1.0.2.0/package/scripts/params_linux.py | 6 --- .../OOZIE/4.0.0.2.0/package/scripts/oozie.py | 6 +-- .../package/scripts/oozie_server_upgrade.py | 7 ++- .../4.0.0.2.0/package/scripts/params_linux.py | 5 +- .../TEZ/0.4.0.2.1/package/scripts/tez_client.py | 32 ------------- .../custom_actions/scripts/remove_bits.py | 2 +- .../BIGTOP/0.8/services/HDFS/metainfo.xml | 16 ------- .../0.8/services/HDFS/package/scripts/params.py | 6 +-- .../BigInsights/4.0/services/HDFS/metainfo.xml | 16 ------- .../4.0/services/HDFS/package/scripts/params.py | 5 -- .../4.0/services/OOZIE/package/scripts/oozie.py | 5 -- .../services/OOZIE/package/scripts/params.py | 19 -------- .../BigInsights/4.1/services/HDFS/metainfo.xml | 13 ----- .../4.2.5/services/HDFS/metainfo.xml | 12 ----- .../4.2.5/services/HDFS/package/scripts/hdfs.py | 5 -- .../HDFS/package/scripts/install_params.py | 7 --- .../HDFS/package/scripts/params_linux.py | 6 --- .../services/OOZIE/package/scripts/oozie.py | 5 +- .../OOZIE/package/scripts/params_linux.py | 3 -- .../BigInsights/4.2/services/HDFS/metainfo.xml | 20 -------- .../4.2/services/HDFS/package/scripts/hdfs.py | 3 -- .../HDFS/package/scripts/install_params.py | 8 ---- .../4.2/services/HDFS/package/scripts/params.py | 9 +--- .../4.2/services/OOZIE/package/scripts/oozie.py | 5 -- .../services/OOZIE/package/scripts/params.py | 19 -------- .../stacks/HDP/2.2/services/HDFS/metainfo.xml | 35 -------------- .../2.3.GlusterFS/services/HDFS/metainfo.xml | 10 ---- .../stacks/HDP/2.3/services/HDFS/metainfo.xml | 30 ------------ .../python/custom_actions/TestRemoveBits.py | 5 +- 39 files changed, 16 insertions(+), 413 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py b/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py index f144b2d..1e388ac 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py @@ -39,7 +39,6 @@ from resource_management.libraries.functions.version import * from resource_management.libraries.functions.format_jvm_option import * from resource_management.libraries.functions.constants import * from resource_management.libraries.functions.get_stack_version import * -from resource_management.libraries.functions.get_lzo_packages import * from resource_management.libraries.functions.setup_ranger_plugin import * from resource_management.libraries.functions.curl_krb_request import * from resource_management.libraries.functions.get_bare_principal import * http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py deleted file mode 100644 index cfbb7d8..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" -__all__ = ["get_lzo_packages"] - -from ambari_commons.os_check import OSCheck -from resource_management.libraries.functions.stack_features import check_stack_feature -from resource_management.libraries.functions import StackFeature -from resource_management.libraries.script.script import Script - -# TODO: Make list of lzo packages stack driven -def get_lzo_packages(stack_version_unformatted): - lzo_packages = [] - script_instance = Script.get_instance() - if OSCheck.is_suse_family() and int(OSCheck.get_os_major_version()) >= 12: - lzo_packages += ["liblzo2-2", "hadoop-lzo-native"] - elif OSCheck.is_redhat_family() or OSCheck.is_suse_family(): - lzo_packages += ["lzo", "hadoop-lzo-native"] - elif OSCheck.is_ubuntu_family(): - lzo_packages += ["liblzo2-2"] - - if stack_version_unformatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_unformatted): - if OSCheck.is_ubuntu_family(): - lzo_packages += [script_instance.format_package_name("hadooplzo-${stack_version}") , - script_instance.format_package_name("hadooplzo-${stack_version}-native")] - else: - lzo_packages += [script_instance.format_package_name("hadooplzo_${stack_version}"), - script_instance.format_package_name("hadooplzo_${stack_version}-native")] - else: - lzo_packages += ["hadoop-lzo"] - - return lzo_packages http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py index 31e78b9..ded63cf 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py @@ -19,7 +19,7 @@ limitations under the License. Ambari Agent """ -__all__ = ["is_lzo_enabled", "should_install_phoenix", "should_install_ams_collector", "should_install_ams_grafana", +__all__ = ["should_install_phoenix", "should_install_ams_collector", "should_install_ams_grafana", "should_install_mysql", "should_install_ranger_tagsync"] import os @@ -44,12 +44,6 @@ def _has_local_components(config, components, indicator_function = any): def _has_applicable_local_component(config, components): return _has_local_components(config, components, any) -def should_install_lzo(): - config = Script.get_config() - io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) - lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() - return lzo_enabled - def should_install_phoenix(): phoenix_hosts = default('/clusterHostInfo/phoenix_query_server_hosts', []) phoenix_enabled = default('/configurations/hbase-env/phoenix_sql_enabled', False) http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml index 2df1b9f..2ab5ef8 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml @@ -203,11 +203,6 @@ <package> <name>hadoop</name> </package> - <package> - <name>hadoop-lzo</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> </packages> </osSpecific> @@ -224,16 +219,6 @@ <name>snappy-devel</name> </package> <package> - <name>lzo</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadoop-lzo-native</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop-libhdfs</name> </package> </packages> @@ -252,16 +237,6 @@ <name>snappy-devel</name> </package> <package> - <name>liblzo2-2</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadoop-lzo-native</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop-libhdfs</name> </package> </packages> @@ -280,11 +255,6 @@ <name>libsnappy-dev</name> </package> <package> - <name>liblzo2-2</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop-hdfs</name> </package> <package> http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py index 8bfa7e6..bbb823d 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py @@ -148,7 +148,6 @@ class DataNodeDefault(DataNode): @OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) class DataNodeWindows(DataNode): def install(self, env): - import install_params self.install_packages(env) if __name__ == "__main__": http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py index 6f702d3..3d6b7d5 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py @@ -133,11 +133,6 @@ def hdfs(name=None): content=Template("slaves.j2") ) - if params.lzo_enabled and len(params.lzo_packages) > 0: - Package(params.lzo_packages, - retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, - retry_count=params.agent_stack_retry_count) - def install_snappy(): import params Directory([params.so_target_dir_x86, params.so_target_dir_x64], http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py index 9c4f9d4..fee0cb5 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py @@ -65,7 +65,6 @@ class HdfsClientDefault(HdfsClient): @OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) class HdfsClientWindows(HdfsClient): def install(self, env): - import install_params self.install_packages(env) self.configure(env) http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py index fe488c3..2f114a7 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py @@ -18,22 +18,13 @@ limitations under the License. """ from ambari_commons import OSCheck +exclude_packages = [] # These parameters are supposed to be referenced at installation time, before the Hadoop environment variables have been set if OSCheck.is_windows_family(): exclude_packages = [] else: from resource_management.libraries.functions.default import default - from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages from resource_management.libraries.script.script import Script _config = Script.get_config() stack_version_unformatted = str(_config['hostLevelParams']['stack_version']) - - # The logic for LZO also exists in OOZIE's params.py - io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) - lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() - lzo_packages = get_lzo_packages(stack_version_unformatted) - - exclude_packages = [] - if not lzo_enabled: - exclude_packages += lzo_packages http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py index 25534e9..bd10289 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py @@ -110,7 +110,6 @@ class JournalNodeDefault(JournalNode): @OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) class JournalNodeWindows(JournalNode): def install(self, env): - import install_params self.install_packages(env) def start(self, env): http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py index 89719cf..95670fe 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py @@ -306,7 +306,6 @@ class NameNodeDefault(NameNode): @OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) class NameNodeWindows(NameNode): def install(self, env): - import install_params self.install_packages(env) #TODO we need this for HA because of manual steps self.configure(env) http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py index 5ca012b..2888b78 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py @@ -40,7 +40,6 @@ from resource_management.libraries.functions.get_not_managed_resources import ge from resource_management.libraries.script.script import Script from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.functions.format_jvm_option import format_jvm_option -from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages from resource_management.libraries.functions.hdfs_utils import is_https_enabled_in_hdfs from resource_management.libraries.functions import is_empty from resource_management.libraries.functions.get_architecture import get_architecture @@ -385,11 +384,6 @@ HdfsResource = functools.partial( ) -# The logic for LZO also exists in OOZIE's params.py -io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) -lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() -lzo_packages = get_lzo_packages(stack_version_unformatted) - name_node_params = default("/commandParams/namenode", None) java_home = config['hostLevelParams']['java_home'] http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py index 7d22603..50d9a4a 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py @@ -305,10 +305,8 @@ def oozie_server_specific(upgrade_type): Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'), not_if = no_op_test) - if params.lzo_enabled and len(params.all_lzo_packages) > 0: - Package(params.all_lzo_packages, - retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, - retry_count=params.agent_stack_retry_count) + # just copying files is ok - we're not making assumptions about installing LZO here + if params.lzo_enabled: Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), not_if = no_op_test, ) http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py index eb57c22..e07616c 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py @@ -61,10 +61,15 @@ class OozieUpgrade(Script): # <stack-selector-tool> set hadoop-client has not run yet, therefore we cannot use # <stack-root>/current/hadoop-client ; we must use params.version directly # however, this only works when upgrading beyond 2.2.0.0; don't do this - # for downgrade to 2.2.0.0 since hadoop-lzo will not be present + # for downgrade to 2.2.0.0 since hadoop-lzo will not be present. + # # This can also be called during a Downgrade. + # # When a version is Installed, it is responsible for downloading the hadoop-lzo packages # if lzo is enabled. + # + # This block is just copying around files - there is no assumption about installation + # if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries): hadoop_lzo_pattern = 'hadoop-lzo*.jar' hadoop_client_new_lib_dir = format("{stack_root}/{version}/hadoop/lib") http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py index cd1f7ad..7c198c4 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py @@ -31,7 +31,6 @@ from resource_management.libraries.functions import get_port_from_url from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster from resource_management.libraries.script.script import Script -from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages from resource_management.libraries.functions.expect import expect from resource_management.libraries.functions.get_architecture import get_architecture from resource_management.libraries.functions.stack_features import get_stack_feature_version @@ -389,6 +388,4 @@ is_webhdfs_enabled = config['configurations']['hdfs-site']['dfs.webhdfs.enabled' # The logic for LZO also exists in HDFS' params.py io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) -lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() - -all_lzo_packages = get_lzo_packages(stack_version_unformatted) +lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py index ed3f5fd..dcf3e24 100644 --- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py +++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py @@ -19,22 +19,15 @@ Ambari Agent """ import os -import urlparse from ambari_commons import OSConst -from ambari_commons.inet_utils import download_file from ambari_commons.os_family_impl import OsFamilyImpl -from ambari_commons.os_utils import copy_file, extract_path_component from resource_management.core.exceptions import ClientComponentHasNoStatus -from resource_management.core.source import InlineTemplate from resource_management.libraries.functions import stack_select from resource_management.libraries.functions import StackFeature from resource_management.libraries.functions.stack_features import check_stack_feature -from resource_management.libraries.functions.get_stack_version import get_stack_version from resource_management.libraries.script.script import Script -from resource_management.libraries.functions.default import default -from resource_management.core.logger import Logger from tez import tez @@ -96,33 +89,8 @@ class TezClientWindows(TezClient): self.install_packages(env) params.refresh_tez_state_dependent_params() env.set_params(params) - self._install_lzo_support_if_needed(params) self.configure(env, config_dir=params.tez_conf_dir) - def _install_lzo_support_if_needed(self, params): - hadoop_classpath_prefix = self._expand_hadoop_classpath_prefix(params.hadoop_classpath_prefix_template, params.config['configurations']['tez-site']) - - hadoop_lzo_dest_path = extract_path_component(hadoop_classpath_prefix, "hadoop-lzo-") - if hadoop_lzo_dest_path: - hadoop_lzo_file = os.path.split(hadoop_lzo_dest_path)[1] - - config = Script.get_config() - file_url = urlparse.urljoin(config['hostLevelParams']['jdk_location'], hadoop_lzo_file) - hadoop_lzo_dl_path = os.path.join(config["hostLevelParams"]["agentCacheDir"], hadoop_lzo_file) - download_file(file_url, hadoop_lzo_dl_path) - #This is for protection against configuration changes. It will infect every new destination with the lzo jar, - # but since the classpath points to the jar directly we're getting away with it. - if not os.path.exists(hadoop_lzo_dest_path): - copy_file(hadoop_lzo_dl_path, hadoop_lzo_dest_path) - - def _expand_hadoop_classpath_prefix(self, hadoop_classpath_prefix_template, configurations): - import resource_management - - hadoop_classpath_prefix_obj = InlineTemplate(hadoop_classpath_prefix_template, configurations_dict=configurations, - extra_imports=[resource_management, resource_management.core, - resource_management.core.source]) - hadoop_classpath_prefix = hadoop_classpath_prefix_obj.get_content() - return hadoop_classpath_prefix if __name__ == "__main__": TezClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/custom_actions/scripts/remove_bits.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/remove_bits.py b/ambari-server/src/main/resources/custom_actions/scripts/remove_bits.py index 0be9a34..078cf32 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/remove_bits.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/remove_bits.py @@ -43,7 +43,7 @@ class RemoveBits(Script): Logger.info("Attempting to remove bits for HDP 2.1") config = Script.get_config() - packages_to_remove = ["zookeeper", "hadoop", "hadoop-lzo", "hadoop-hdfs", "hadoop-libhdfs", "hadoop-yarn", "hadoop-client", "hadoop-mapreduce", "hive", "hive-hcatalog", "hive-jdbc", "hive-webhcat", "hcatalog", "webhcat-tar-hive", "webhcat-tar-pig", "oozie", "oozie-client", "pig", "sqoop", "tez" "falcon", "storm", "flume", "hbase", "phoenix"] + packages_to_remove = ["zookeeper", "hadoop", "hadoop-hdfs", "hadoop-libhdfs", "hadoop-yarn", "hadoop-client", "hadoop-mapreduce", "hive", "hive-hcatalog", "hive-jdbc", "hive-webhcat", "hcatalog", "webhcat-tar-hive", "webhcat-tar-pig", "oozie", "oozie-client", "pig", "sqoop", "tez" "falcon", "storm", "flume", "hbase", "phoenix"] packages_to_remove.reverse() Logger.info("Packages to remove: {0}".format(" ".join(packages_to_remove))) http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/metainfo.xml index 8500839..f473598 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/metainfo.xml @@ -180,10 +180,6 @@ <package> <name>hadoop</name> </package> - <package> - <name>hadoop-lzo</name> - <skipUpgrade>true</skipUpgrade> - </package> </packages> </osSpecific> @@ -197,14 +193,6 @@ <name>snappy-devel</name> </package> <package> - <name>lzo</name> - <skipUpgrade>true</skipUpgrade> - </package> - <package> - <name>hadoop-lzo-native</name> - <skipUpgrade>true</skipUpgrade> - </package> - <package> <name>hadoop-libhdfs</name> </package> </packages> @@ -220,10 +208,6 @@ <name>libsnappy-dev</name> </package> <package> - <name>liblzo2-2</name> - <skipUpgrade>true</skipUpgrade> - </package> - <package> <name>hadoop-hdfs</name> </package> <package> http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py index 3a711c7..f16242c 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py @@ -209,11 +209,7 @@ HdfsDirectory = functools.partial( bin_dir = hadoop_bin_dir ) -io_compression_codecs = config['configurations']['core-site']['io.compression.codecs'] -if not "com.hadoop.compression.lzo" in io_compression_codecs: - exclude_packages = ["lzo", "hadoop-lzo", "hadoop-lzo-native", "liblzo2-2"] -else: - exclude_packages = [] +exclude_packages = [] name_node_params = default("/commandParams/namenode", None) #hadoop params http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml index 918cdb3..1efd746 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml @@ -152,10 +152,6 @@ <package> <name>hadoop</name> </package> - <package> - <name>hadoop-lzo</name> - <condition>should_install_lzo</condition> - </package> </packages> </osSpecific> @@ -166,14 +162,6 @@ <name>snappy</name> </package> <package> - <name>lzo</name> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadoop-lzo-native</name> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop-libhdfs</name> </package> <package> @@ -192,10 +180,6 @@ <name>libsnappy-dev</name> </package> <package> - <name>liblzo2-2</name> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop-hdfs</name> </package> <package> http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/package/scripts/params.py index 1f76f80..d604906 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/package/scripts/params.py @@ -293,11 +293,6 @@ HdfsResource = functools.partial( default_fs = default_fs ) -io_compression_codecs = config['configurations']['core-site']['io.compression.codecs'] -if not "com.hadoop.compression.lzo" in io_compression_codecs: - exclude_packages = ["lzo", "hadoop-lzo", "hadoop-lzo-native", "liblzo2-2"] -else: - exclude_packages = [] name_node_params = default("/commandParams/namenode", None) #hadoop params http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/oozie.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/oozie.py index 6cd2b71..76a2abe 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/oozie.py @@ -227,11 +227,6 @@ def oozie_server_specific(): Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension.jar'), not_if = no_op_test, ) - #if params.lzo_enabled and len(params.lzo_packages_for_current_host) > 0: - # Package(params.lzo_packages_for_current_host) - # Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), - # not_if = no_op_test, - # ) prepare_war_cmd_file = format("{oozie_home}/.prepare_war_cmd") prepare_war_cmd = format("cd {oozie_tmp_dir} && {oozie_setup_sh} prepare-war {oozie_secure}") http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/params.py index febd3e6..17ea124 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/params.py @@ -235,25 +235,6 @@ HdfsResource = functools.partial( #-----LZO is not suppported in IOP distribution since it is GPL license-------- -''' io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() -# stack_is_iop40_or_further -underscored_version = stack_version_unformatted.replace('.', '_') -dashed_version = stack_version_unformatted.replace('.', '-') -lzo_packages_to_family = { - "any": ["hadoop-lzo", ], - "redhat": ["lzo", "hadoop-lzo-native"], - "suse": ["lzo", "hadoop-lzo-native"], - "ubuntu": ["liblzo2-2", ] -} - - -lzo_packages_to_family["redhat"] += [format("hadooplzo_{underscorred_version}_*")] -lzo_packages_to_family["suse"] += [format("hadooplzo_{underscorred_version}_*")] -lzo_packages_to_family["ubuntu"] += [format("hadooplzo_{dashed_version}_*")] - -lzo_packages_for_current_host = lzo_packages_to_family['any'] + lzo_packages_to_family[System.get_instance().os_family] -all_lzo_packages = set(itertools.chain(*lzo_packages_to_family.values())) -''' http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/stacks/BigInsights/4.1/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.1/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.1/services/HDFS/metainfo.xml index 4c00b4f..45b16ee 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.1/services/HDFS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.1/services/HDFS/metainfo.xml @@ -62,10 +62,6 @@ <package> <name>hadoop_4_1_*</name> </package> - <package> - <name>hadoop-lzo</name> - <!--name>hadoop_4_1_*-lzo</name--> - </package> </packages> </osSpecific> @@ -76,12 +72,6 @@ <name>snappy</name> </package> <package> - <name>lzo</name> - </package> - <package> - <name>hadoop-lzo-native</name> - </package> - <package> <name>hadoop_4_1_*-libhdfs</name> </package> <package> @@ -100,9 +90,6 @@ <name>libsnappy-dev</name> </package> <package> - <name>liblzo2-2</name> - </package> - <package> <name>hadoop-hdfs</name> </package> <package> http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/metainfo.xml index fc7c324..95724b7 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/metainfo.xml @@ -63,9 +63,6 @@ <package> <name>hadoop_4_2_5_*</name> </package> - <package> - <name>hadoop-lzo</name> - </package> </packages> </osSpecific> <osSpecific> @@ -75,12 +72,6 @@ <name>snappy</name> </package> <package> - <name>lzo</name> - </package> - <package> - <name>hadoop-lzo-native</name> - </package> - <package> <name>hadoop_4_2_5_*-libhdfs</name> </package> <package> @@ -104,9 +95,6 @@ <name>libsnappy-dev</name> </package> <package> - <name>liblzo2-2</name> - </package> - <package> <name>hadoop-hdfs</name> </package> <package> http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/hdfs.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/hdfs.py index 1264284..096c4d7 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/hdfs.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/hdfs.py @@ -116,11 +116,6 @@ def hdfs(name=None): content=Template("slaves.j2") ) - if params.lzo_enabled and len(params.lzo_packages) > 0: - Package(params.lzo_packages, - retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, - retry_count=params.agent_stack_retry_count) - def install_snappy(): import params Directory([params.so_target_dir_x86, params.so_target_dir_x64], http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/install_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/install_params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/install_params.py index da8de46..9d2c4f3 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/install_params.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/install_params.py @@ -28,11 +28,4 @@ else: _config = Script.get_config() stack_version_unformatted = str(_config['hostLevelParams']['stack_version']) - # The logic for LZO also exists in OOZIE's params.py - io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) - lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() - lzo_packages = ["lzo", "hadoop-lzo", "hadoop-lzo-native"] - exclude_packages = [] - if not lzo_enabled: - exclude_packages += lzo_packages http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/params_linux.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/params_linux.py index a77ec82..37c2b85 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/params_linux.py @@ -362,12 +362,6 @@ HdfsResource = functools.partial( dfs_type = dfs_type ) - -# The logic for LZO also exists in OOZIE's params.py -io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) -lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() -lzo_packages = ["lzo", "hadoop-lzo", "hadoop-lzo-native"] - name_node_params = default("/commandParams/namenode", None) java_home = config['hostLevelParams']['java_home'] http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie.py index 33a309f..5f572a7 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie.py @@ -305,10 +305,7 @@ def oozie_server_specific(upgrade_type): Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'), not_if = no_op_test) - if params.lzo_enabled and len(params.all_lzo_packages) > 0: - Package(params.all_lzo_packages, - retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, - retry_count=params.agent_stack_retry_count) + if params.lzo_enabled: Execute(format('{sudo} cp /usr/lib/hadoop-lzo/lib/hadoop-lzo*.jar {oozie_lib_dir}'), not_if = no_op_test, ) http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/params_linux.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/params_linux.py index 0e6179b..f87e3db 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/params_linux.py @@ -30,7 +30,6 @@ from resource_management.libraries.functions import get_port_from_url from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster from resource_management.libraries.script.script import Script -from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages from resource_management.libraries.functions.expect import expect from resource_management.libraries.functions.get_architecture import get_architecture from resource_management.libraries.functions.stack_features import get_stack_feature_version @@ -383,5 +382,3 @@ is_webhdfs_enabled = config['configurations']['hdfs-site']['dfs.webhdfs.enabled' # The logic for LZO also exists in HDFS' params.py io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() - -all_lzo_packages = ["lzo", "hadoop-lzo", "hadoop-lzo-native"] http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml index 72ce2be..4253eb1 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml @@ -182,11 +182,6 @@ <package> <name>hadoop_4_2_*</name> </package> - <package> - <name>hadoop-lzo</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> </packages> </osSpecific> <osSpecific> @@ -196,16 +191,6 @@ <name>snappy</name> </package> <package> - <name>lzo</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadoop-lzo-native</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop_4_2_*-libhdfs</name> </package> <package> @@ -229,11 +214,6 @@ <name>libsnappy-dev</name> </package> <package> - <name>liblzo2-2</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop-hdfs</name> </package> <package> http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py index 2b158bd..cf54f1f 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py @@ -115,9 +115,6 @@ def hdfs(name=None): content=Template("slaves.j2") ) - if params.lzo_enabled and len(params.lzo_packages) > 0: - Package(params.lzo_packages) - def install_snappy(): import params Directory([params.so_target_dir_x86, params.so_target_dir_x64], http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/install_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/install_params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/install_params.py index fe488c3..9d2c4f3 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/install_params.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/install_params.py @@ -23,17 +23,9 @@ if OSCheck.is_windows_family(): exclude_packages = [] else: from resource_management.libraries.functions.default import default - from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages from resource_management.libraries.script.script import Script _config = Script.get_config() stack_version_unformatted = str(_config['hostLevelParams']['stack_version']) - # The logic for LZO also exists in OOZIE's params.py - io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) - lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() - lzo_packages = get_lzo_packages(stack_version_unformatted) - exclude_packages = [] - if not lzo_enabled: - exclude_packages += lzo_packages http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/params.py index 7b9c449..cd426b3 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/params.py @@ -304,14 +304,7 @@ HdfsResource = functools.partial( default_fs = default_fs ) -lzo_enabled = True -lzo_packages = ["lzo", "hadoop-lzo", "hadoop-lzo-native"] -io_compression_codecs = config['configurations']['core-site']['io.compression.codecs'] -if not "com.hadoop.compression.lzo" in io_compression_codecs: - lzo_enabled = False - exclude_packages = ["lzo", "hadoop-lzo", "hadoop-lzo-native", "liblzo2-2"] -else: - exclude_packages = [] + name_node_params = default("/commandParams/namenode", '{"threshold":"10"}') #hadoop params http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py index 64a83c0..ac69d37 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py @@ -227,11 +227,6 @@ def oozie_server_specific(): Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension.jar'), not_if = no_op_test, ) - #if params.lzo_enabled and len(params.lzo_packages_for_current_host) > 0: - # Package(params.lzo_packages_for_current_host) - # Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), - # not_if = no_op_test, - # ) prepare_war_cmd_file = format("{oozie_home}/.prepare_war_cmd") prepare_war_cmd = format("cd {oozie_tmp_dir} && {oozie_setup_sh} prepare-war {oozie_secure}") http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py index d1a4a66..a3b0c3f 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py @@ -238,25 +238,6 @@ HdfsResource = functools.partial( #-----LZO is not suppported in IOP distribution since it is GPL license-------- -''' io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() -# stack_is_iop40_or_further -underscored_version = stack_version_unformatted.replace('.', '_') -dashed_version = stack_version_unformatted.replace('.', '-') -lzo_packages_to_family = { - "any": ["hadoop-lzo", ], - "redhat": ["lzo", "hadoop-lzo-native"], - "suse": ["lzo", "hadoop-lzo-native"], - "ubuntu": ["liblzo2-2", ] -} - - -lzo_packages_to_family["redhat"] += [format("hadooplzo_{underscorred_version}_*")] -lzo_packages_to_family["suse"] += [format("hadooplzo_{underscorred_version}_*")] -lzo_packages_to_family["ubuntu"] += [format("hadooplzo_{dashed_version}_*")] - -lzo_packages_for_current_host = lzo_packages_to_family['any'] + lzo_packages_to_family[System.get_instance().os_family] -all_lzo_packages = set(itertools.chain(*lzo_packages_to_family.values())) -''' http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml index 07a9ef8..8aa69a9 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml @@ -40,19 +40,6 @@ <name>snappy-devel</name> </package> <package> - <name>lzo</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}</name> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}-native</name> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop_${stack_version}-libhdfs</name> </package> </packages> @@ -74,19 +61,6 @@ <name>snappy-devel</name> </package> <package> - <name>liblzo2-2</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}</name> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}-native</name> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop_${stack_version}-libhdfs</name> </package> </packages> @@ -120,15 +94,6 @@ <name>libsnappy-dev</name> </package> <package> - <name>hadooplzo-${stack_version}</name> - <condition>should_install_lzo</condition> - </package> - <package> - <name>liblzo2-2</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> <name>libhdfs0-${stack_version}</name> </package> </packages> http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/HDFS/metainfo.xml index 15fe931..6a9fab61 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/HDFS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/HDFS/metainfo.xml @@ -36,13 +36,6 @@ <name>snappy-devel</name> </package> <package> - <name>lzo</name> - <skipUpgrade>true</skipUpgrade> - </package> - <package> - <name>hadooplzo_2_3_*</name> - </package> - <package> <name>hadoop_2_3_*-libhdfs</name> </package> </packages> @@ -76,9 +69,6 @@ <name>libsnappy-dev</name> </package> <package> - <name>hadooplzo-2-3-.*</name> - </package> - <package> <name>libhdfs0-2-3-.*</name> </package> </packages> http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml index 63e85e3..e02bf6b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml @@ -72,19 +72,6 @@ <name>snappy-devel</name> </package> <package> - <name>lzo</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}</name> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}-native</name> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop_${stack_version}-libhdfs</name> </package> </packages> @@ -106,19 +93,6 @@ <name>snappy-devel</name> </package> <package> - <name>liblzo2-2</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}</name> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}-native</name> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop_${stack_version}-libhdfs</name> </package> </packages> @@ -152,10 +126,6 @@ <name>libsnappy-dev</name> </package> <package> - <name>hadooplzo-${stack_version}</name> - <condition>should_install_lzo</condition> - </package> - <package> <name>libhdfs0-${stack_version}</name> </package> </packages> http://git-wip-us.apache.org/repos/asf/ambari/blob/6df33612/ambari-server/src/test/python/custom_actions/TestRemoveBits.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/custom_actions/TestRemoveBits.py b/ambari-server/src/test/python/custom_actions/TestRemoveBits.py index 6537152..8217ca0 100644 --- a/ambari-server/src/test/python/custom_actions/TestRemoveBits.py +++ b/ambari-server/src/test/python/custom_actions/TestRemoveBits.py @@ -105,13 +105,10 @@ class TestRemoveBits(RMFTestCase): self.assertResourceCalled('Package', 'hadoop-hdfs', action = ['remove'], ) - self.assertResourceCalled('Package', 'hadoop-lzo', - action = ['remove'], - ) self.assertResourceCalled('Package', 'hadoop', action = ['remove'], ) self.assertResourceCalled('Package', 'zookeeper', action = ['remove'], ) - self.assertNoMoreResources() \ No newline at end of file + self.assertNoMoreResources()
