Repository: ambari Updated Branches: refs/heads/trunk 135d88bc1 -> 7eb14a0a5
AMBARI-22467. YARN, MapReduce2, Hive, and Oozie Should Conditionally Install LZO (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7eb14a0a Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7eb14a0a Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7eb14a0a Branch: refs/heads/trunk Commit: 7eb14a0a563eb478d3e47d910c4dbf66b05fdfdf Parents: 135d88b Author: Andrew Onishuk <[email protected]> Authored: Wed Nov 22 13:45:30 2017 +0200 Committer: Andrew Onishuk <[email protected]> Committed: Wed Nov 22 13:45:30 2017 +0200 ---------------------------------------------------------------------- .../libraries/functions/__init__.py | 2 +- .../libraries/functions/get_lzo_packages.py | 50 ----------- .../libraries/functions/lzo_utils.py | 93 ++++++++++++++++++++ .../libraries/functions/package_conditions.py | 8 +- .../common-services/HDFS/2.1.0.2.0/metainfo.xml | 30 ------- .../HDFS/2.1.0.2.0/package/scripts/hdfs.py | 7 +- .../2.1.0.2.0/package/scripts/install_params.py | 4 - .../2.1.0.2.0/package/scripts/params_linux.py | 5 -- .../common-services/HDFS/3.0.0.3.0/metainfo.xml | 30 ------- .../HDFS/3.0.0.3.0/package/scripts/hdfs.py | 7 +- .../3.0.0.3.0/package/scripts/install_params.py | 4 - .../3.0.0.3.0/package/scripts/params_linux.py | 4 - .../HIVE/0.12.0.2.0/package/scripts/hive.py | 3 + .../HIVE/2.1.0.3.0/package/scripts/hive.py | 3 + .../OOZIE/4.0.0.2.0/package/scripts/oozie.py | 38 ++++---- .../4.0.0.2.0/package/scripts/params_linux.py | 5 +- .../OOZIE/4.2.0.3.0/package/scripts/oozie.py | 16 ++-- .../4.2.0.3.0/package/scripts/params_linux.py | 5 +- .../YARN/2.1.0.2.0/package/scripts/yarn.py | 3 + .../YARN/3.0.0.3.0/package/scripts/yarn.py | 3 + .../stacks/HDP/2.2/services/HDFS/metainfo.xml | 35 -------- .../stacks/HDP/2.3/services/HDFS/metainfo.xml | 30 ------- .../stacks/HDP/3.0/services/HDFS/metainfo.xml | 30 ------- .../stacks/2.2/configs/oozie-upgrade.json | 3 +- 24 files changed, 142 insertions(+), 276 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/7eb14a0a/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py b/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py index f144b2d..b907844 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py @@ -39,7 +39,7 @@ from resource_management.libraries.functions.version import * from resource_management.libraries.functions.format_jvm_option import * from resource_management.libraries.functions.constants import * from resource_management.libraries.functions.get_stack_version import * -from resource_management.libraries.functions.get_lzo_packages import * +from resource_management.libraries.functions.lzo_utils import * from resource_management.libraries.functions.setup_ranger_plugin import * from resource_management.libraries.functions.curl_krb_request import * from resource_management.libraries.functions.get_bare_principal import * http://git-wip-us.apache.org/repos/asf/ambari/blob/7eb14a0a/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py deleted file mode 100644 index cfbb7d8..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" -__all__ = ["get_lzo_packages"] - -from ambari_commons.os_check import OSCheck -from resource_management.libraries.functions.stack_features import check_stack_feature -from resource_management.libraries.functions import StackFeature -from resource_management.libraries.script.script import Script - -# TODO: Make list of lzo packages stack driven -def get_lzo_packages(stack_version_unformatted): - lzo_packages = [] - script_instance = Script.get_instance() - if OSCheck.is_suse_family() and int(OSCheck.get_os_major_version()) >= 12: - lzo_packages += ["liblzo2-2", "hadoop-lzo-native"] - elif OSCheck.is_redhat_family() or OSCheck.is_suse_family(): - lzo_packages += ["lzo", "hadoop-lzo-native"] - elif OSCheck.is_ubuntu_family(): - lzo_packages += ["liblzo2-2"] - - if stack_version_unformatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_unformatted): - if OSCheck.is_ubuntu_family(): - lzo_packages += [script_instance.format_package_name("hadooplzo-${stack_version}") , - script_instance.format_package_name("hadooplzo-${stack_version}-native")] - else: - lzo_packages += [script_instance.format_package_name("hadooplzo_${stack_version}"), - script_instance.format_package_name("hadooplzo_${stack_version}-native")] - else: - lzo_packages += ["hadoop-lzo"] - - return lzo_packages http://git-wip-us.apache.org/repos/asf/ambari/blob/7eb14a0a/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py b/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py new file mode 100644 index 0000000..d6d987f --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +__all__ = ["should_install_lzo", "get_lzo_packages", "install_lzo_if_needed"] + +from ambari_commons.os_check import OSCheck +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions import StackFeature, stack_features +from resource_management.libraries.script.script import Script +from resource_management.core.logger import Logger +from resource_management.libraries.functions.expect import expect +from resource_management.core.resources.packaging import Package + +INSTALLING_LZO_WITHOUT_GPL = "Cannot install LZO. The GPL license must be explicitly enabled using 'ambari-server setup' on the Ambari host, then restart the server and try again." + +def get_lzo_packages(): + lzo_packages = [] + script_instance = Script.get_instance() + if OSCheck.is_suse_family() and int(OSCheck.get_os_major_version()) >= 12: + lzo_packages += ["liblzo2-2", "hadoop-lzo-native"] + elif OSCheck.is_redhat_family() or OSCheck.is_suse_family(): + lzo_packages += ["lzo", "hadoop-lzo-native"] + elif OSCheck.is_ubuntu_family(): + lzo_packages += ["liblzo2-2"] + + + stack_version_unformatted = stack_features.get_stack_feature_version(Script.get_config()) # only used to check stack_feature, NOT as package version! + if stack_version_unformatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_unformatted): + if OSCheck.is_ubuntu_family(): + lzo_packages += [script_instance.format_package_name("hadooplzo-${stack_version}") , + script_instance.format_package_name("hadooplzo-${stack_version}-native")] + else: + lzo_packages += [script_instance.format_package_name("hadooplzo_${stack_version}"), + script_instance.format_package_name("hadooplzo_${stack_version}-native")] + else: + lzo_packages += ["hadoop-lzo"] + + return lzo_packages + +def should_install_lzo(): + """ + Return true if lzo is enabled via core-site.xml and GPL license (required for lzo) is accepted. + """ + config = Script.get_config() + io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) + lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() + + if not lzo_enabled: + return False + + is_gpl_license_accepted = default("/hostLevelParams/gpl_license_accepted", False) + if not is_gpl_license_accepted: + Logger.warning(INSTALLING_LZO_WITHOUT_GPL) + return False + + return True + +def install_lzo_if_needed(): + """ + Install lzo package if {#should_install_lzo} is true + """ + if not should_install_lzo(): + return + + lzo_packages = get_lzo_packages() + + config = Script.get_config() + agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_retry_on_unavailability'] + agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int) + + Package(lzo_packages, + retry_on_repo_unavailability=agent_stack_retry_on_unavailability, + retry_count=agent_stack_retry_count + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/7eb14a0a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py index 31e78b9..ded63cf 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py @@ -19,7 +19,7 @@ limitations under the License. Ambari Agent """ -__all__ = ["is_lzo_enabled", "should_install_phoenix", "should_install_ams_collector", "should_install_ams_grafana", +__all__ = ["should_install_phoenix", "should_install_ams_collector", "should_install_ams_grafana", "should_install_mysql", "should_install_ranger_tagsync"] import os @@ -44,12 +44,6 @@ def _has_local_components(config, components, indicator_function = any): def _has_applicable_local_component(config, components): return _has_local_components(config, components, any) -def should_install_lzo(): - config = Script.get_config() - io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) - lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() - return lzo_enabled - def should_install_phoenix(): phoenix_hosts = default('/clusterHostInfo/phoenix_query_server_hosts', []) phoenix_enabled = default('/configurations/hbase-env/phoenix_sql_enabled', False) http://git-wip-us.apache.org/repos/asf/ambari/blob/7eb14a0a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml index 9979de4..6bbb583 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml @@ -244,11 +244,6 @@ <package> <name>hadoop</name> </package> - <package> - <name>hadoop-lzo</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> </packages> </osSpecific> @@ -265,16 +260,6 @@ <name>snappy-devel</name> </package> <package> - <name>lzo</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadoop-lzo-native</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop-libhdfs</name> </package> <package> @@ -296,16 +281,6 @@ <name>snappy-devel</name> </package> <package> - <name>liblzo2-2</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadoop-lzo-native</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop-libhdfs</name> </package> </packages> @@ -324,11 +299,6 @@ <name>libsnappy-dev</name> </package> <package> - <name>liblzo2-2</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop-hdfs</name> </package> <package> http://git-wip-us.apache.org/repos/asf/ambari/blob/7eb14a0a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py index 4022986..bc80ba6 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py @@ -33,6 +33,7 @@ from resource_management.libraries.functions.format import format import os from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from ambari_commons import OSConst +from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) def hdfs(name=None): @@ -143,11 +144,7 @@ def hdfs(name=None): content=Template("slaves.j2") ) - if params.lzo_enabled: - lzo_packages = get_lzo_packages(params.stack_version_unformatted) - Package(lzo_packages, - retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, - retry_count=params.agent_stack_retry_count) + install_lzo_if_needed() def install_snappy(): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/7eb14a0a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py index 235f231..dc3279f 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py @@ -27,7 +27,3 @@ else: _config = Script.get_config() stack_version_unformatted = str(_config['hostLevelParams']['stack_version']) - - # The logic for LZO also exists in OOZIE's params.py - io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) - lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() http://git-wip-us.apache.org/repos/asf/ambari/blob/7eb14a0a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py index bb6349b..e2790e1 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py @@ -383,11 +383,6 @@ HdfsResource = functools.partial( immutable_paths = get_not_managed_resources(), dfs_type = dfs_type ) - - -# The logic for LZO also exists in OOZIE's params.py -io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) -lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() name_node_params = default("/commandParams/namenode", None) http://git-wip-us.apache.org/repos/asf/ambari/blob/7eb14a0a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/metainfo.xml b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/metainfo.xml index e6d1166..0c629f3 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/metainfo.xml @@ -270,11 +270,6 @@ <package> <name>hadoop</name> </package> - <package> - <name>hadoop-lzo</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> </packages> </osSpecific> @@ -291,16 +286,6 @@ <name>snappy-devel</name> </package> <package> - <name>lzo</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadoop-lzo-native</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop-libhdfs</name> </package> </packages> @@ -319,16 +304,6 @@ <name>snappy-devel</name> </package> <package> - <name>liblzo2-2</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadoop-lzo-native</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop-libhdfs</name> </package> </packages> @@ -347,11 +322,6 @@ <name>libsnappy-dev</name> </package> <package> - <name>liblzo2-2</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop-hdfs</name> </package> <package> http://git-wip-us.apache.org/repos/asf/ambari/blob/7eb14a0a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs.py index 4022986..bc80ba6 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs.py +++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs.py @@ -33,6 +33,7 @@ from resource_management.libraries.functions.format import format import os from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from ambari_commons import OSConst +from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) def hdfs(name=None): @@ -143,11 +144,7 @@ def hdfs(name=None): content=Template("slaves.j2") ) - if params.lzo_enabled: - lzo_packages = get_lzo_packages(params.stack_version_unformatted) - Package(lzo_packages, - retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, - retry_count=params.agent_stack_retry_count) + install_lzo_if_needed() def install_snappy(): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/7eb14a0a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/install_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/install_params.py b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/install_params.py index 235f231..dc3279f 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/install_params.py +++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/install_params.py @@ -27,7 +27,3 @@ else: _config = Script.get_config() stack_version_unformatted = str(_config['hostLevelParams']['stack_version']) - - # The logic for LZO also exists in OOZIE's params.py - io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) - lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() http://git-wip-us.apache.org/repos/asf/ambari/blob/7eb14a0a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py index 2fa6208..cbe7943 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py @@ -373,10 +373,6 @@ HdfsResource = functools.partial( dfs_type = dfs_type ) - -# The logic for LZO also exists in OOZIE's params.py -io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) -lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() name_node_params = default("/commandParams/namenode", None) http://git-wip-us.apache.org/repos/asf/ambari/blob/7eb14a0a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py index c4b34a5..716a37c 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py @@ -42,6 +42,7 @@ from resource_management.core.logger import Logger from resource_management.core import utils from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster, setup_atlas_hook from resource_management.libraries.functions.security_commons import update_credential_provider_path +from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed from ambari_commons.constants import SERVICE from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl @@ -50,6 +51,8 @@ from ambari_commons import OSConst @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) def hive(name=None): import params + + install_lzo_if_needed() hive_client_conf_path = format("{stack_root}/current/{component_directory}/conf") # Permissions 644 for conf dir (client) files, and 600 for conf.server http://git-wip-us.apache.org/repos/asf/ambari/blob/7eb14a0a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py index 22ff9fd..1724bae 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py @@ -42,6 +42,7 @@ from resource_management.core.logger import Logger from resource_management.core import utils from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster, setup_atlas_hook from resource_management.libraries.functions.security_commons import update_credential_provider_path +from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed from ambari_commons.constants import SERVICE from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl @@ -50,6 +51,8 @@ from ambari_commons import OSConst @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) def hive(name=None): import params + + install_lzo_if_needed() hive_client_conf_path = format("{stack_root}/current/{component_directory}/conf") # Permissions 644 for conf dir (client) files, and 600 for conf.server http://git-wip-us.apache.org/repos/asf/ambari/blob/7eb14a0a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py index f215a1e..2306108 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py @@ -35,6 +35,7 @@ from resource_management.libraries.functions.stack_features import check_stack_f from resource_management.libraries.functions.oozie_prepare_war import prepare_war from resource_management.libraries.functions.copy_tarball import get_current_version from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed from resource_management.libraries.script.script import Script from resource_management.libraries.functions.security_commons import update_credential_provider_path from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages @@ -190,6 +191,12 @@ def oozie(is_server=False, upgrade_type=None): oozie_ownership() + if params.lzo_enabled: + install_lzo_if_needed() + Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), + not_if = no_op_test, + ) + if is_server: oozie_server_specific(upgrade_type) @@ -238,14 +245,14 @@ def get_oozie_ext_zip_source_paths(upgrade_type, params): def oozie_server_specific(upgrade_type): import params - + no_op_test = as_user(format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"), user=params.oozie_user) - + File(params.pid_file, action="delete", not_if=no_op_test ) - + oozie_server_directories = [format("{oozie_home}/{oozie_tmp_dir}"), params.oozie_pid_dir, params.oozie_log_dir, params.oozie_tmp_dir, params.oozie_data_dir, params.oozie_lib_dir, params.oozie_webapps_dir, params.oozie_webapps_conf_dir, params.oozie_server_dir] Directory( oozie_server_directories, owner = params.oozie_user, @@ -254,25 +261,25 @@ def oozie_server_specific(upgrade_type): create_parents = True, cd_access="a", ) - + Directory(params.oozie_libext_dir, create_parents = True, ) - + hashcode_file = format("{oozie_home}/.hashcode") skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share") untar_sharelib = ('tar','-xvf',format('{oozie_home}/oozie-sharelib.tar.gz'),'-C',params.oozie_home) Execute( untar_sharelib, # time-expensive - not_if = format("{no_op_test} || {skip_recreate_sharelib}"), + not_if = format("{no_op_test} || {skip_recreate_sharelib}"), sudo = True, ) configure_cmds = [] # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path source_ext_zip_paths = get_oozie_ext_zip_source_paths(upgrade_type, params) - + # Copy the first oozie ext-2.2.zip file that is found. # This uses a list to handle the cases when migrating from some versions of BigInsights to HDP. if source_ext_zip_paths is not None: @@ -286,8 +293,8 @@ def oozie_server_specific(upgrade_type): sudo=True, ) break - - + + Directory(params.oozie_webapps_conf_dir, owner = params.oozie_user, group = params.user_group, @@ -306,15 +313,6 @@ def oozie_server_specific(upgrade_type): Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'), not_if = no_op_test) - if params.lzo_enabled: - all_lzo_packages = get_lzo_packages(params.stack_version_unformatted) - Package(all_lzo_packages, - retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, - retry_count=params.agent_stack_retry_count) - Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), - not_if = no_op_test, - ) - prepare_war(params) File(hashcode_file, @@ -365,7 +363,7 @@ def oozie_server_specific(upgrade_type): Directory(params.oozie_server_dir, owner = params.oozie_user, group = params.user_group, - recursive_ownership = True, + recursive_ownership = True, ) if params.security_enabled: File(os.path.join(params.conf_dir, 'zkmigrator_jaas.conf'), @@ -410,7 +408,7 @@ def copy_atlas_hive_hook_to_dfs_share_lib(upgrade_type=None, upgrade_direction=N effective_version = params.stack_version_formatted if upgrade_type is None else format_stack_version(params.version) if not check_stack_feature(StackFeature.ATLAS_HOOK_SUPPORT, effective_version): return - + # Important that oozie_server_hostnames is sorted by name so that this only runs on a single Oozie server. if not (len(params.oozie_server_hostnames) > 0 and params.hostname == params.oozie_server_hostnames[0]): Logger.debug("Will not attempt to copy Atlas Hive hook to DFS since this is not the first Oozie Server " http://git-wip-us.apache.org/repos/asf/ambari/blob/7eb14a0a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py index a0f0672..125ecfe 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py @@ -30,6 +30,7 @@ from resource_management.libraries.functions import get_port_from_url from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.lzo_utils import should_install_lzo from resource_management.libraries.functions.expect import expect from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.functions.get_architecture import get_architecture @@ -384,6 +385,4 @@ HdfsResource = functools.partial( is_webhdfs_enabled = config['configurations']['hdfs-site']['dfs.webhdfs.enabled'] -# The logic for LZO also exists in HDFS' params.py -io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) -lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() +lzo_enabled = should_install_lzo() http://git-wip-us.apache.org/repos/asf/ambari/blob/7eb14a0a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py index 0771e93..1db16a9 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py @@ -35,6 +35,7 @@ from resource_management.libraries.functions.stack_features import check_stack_f from resource_management.libraries.functions.oozie_prepare_war import prepare_war from resource_management.libraries.functions.copy_tarball import get_current_version from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed from resource_management.libraries.script.script import Script from resource_management.libraries.functions.security_commons import update_credential_provider_path from resource_management.core.resources.packaging import Package @@ -189,6 +190,12 @@ def oozie(is_server=False): oozie_ownership() + if params.lzo_enabled: + install_lzo_if_needed() + Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), + not_if = no_op_test, + ) + if is_server: oozie_server_specific() @@ -275,15 +282,6 @@ def oozie_server_specific(): Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'), not_if = no_op_test) - if params.lzo_enabled: - all_lzo_packages = get_lzo_packages(params.stack_version_unformatted) - Package(all_lzo_packages, - retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, - retry_count=params.agent_stack_retry_count) - Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), - not_if = no_op_test, - ) - prepare_war(params) File(hashcode_file, http://git-wip-us.apache.org/repos/asf/ambari/blob/7eb14a0a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/params_linux.py index 70b89b7..1ae86e8 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/params_linux.py @@ -28,6 +28,7 @@ from resource_management.libraries.functions import get_port_from_url from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.lzo_utils import should_install_lzo from resource_management.libraries.functions.expect import expect from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.functions.get_architecture import get_architecture @@ -366,6 +367,4 @@ HdfsResource = functools.partial( is_webhdfs_enabled = config['configurations']['hdfs-site']['dfs.webhdfs.enabled'] -# The logic for LZO also exists in HDFS' params.py -io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) -lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() +lzo_enabled = should_install_lzo() http://git-wip-us.apache.org/repos/asf/ambari/blob/7eb14a0a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py index 363d464..d077c84 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py @@ -27,6 +27,7 @@ from resource_management.libraries.script.script import Script from resource_management.core.resources.service import ServiceConfig from resource_management.libraries.functions.format import format from resource_management.libraries.functions.is_empty import is_empty +from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed from resource_management.core.resources.system import Directory from resource_management.core.resources.system import File from resource_management.libraries.resources.xml_config import XmlConfig @@ -45,6 +46,8 @@ def yarn(name=None, config_dir=None): """ import params + install_lzo_if_needed() + if config_dir is None: config_dir = params.hadoop_conf_dir http://git-wip-us.apache.org/repos/asf/ambari/blob/7eb14a0a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py index ef314f3..7731aeb 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py @@ -27,6 +27,7 @@ from resource_management.libraries.script.script import Script from resource_management.core.resources.service import ServiceConfig from resource_management.libraries.functions.format import format from resource_management.libraries.functions.is_empty import is_empty +from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed from resource_management.core.resources.system import Directory from resource_management.core.resources.system import File from resource_management.libraries.resources.xml_config import XmlConfig @@ -44,6 +45,8 @@ def yarn(name=None, config_dir=None): :param config_dir: Which config directory to write configs to, which could be different during rolling upgrade. """ import params + + install_lzo_if_needed() if config_dir is None: config_dir = params.hadoop_conf_dir http://git-wip-us.apache.org/repos/asf/ambari/blob/7eb14a0a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml index 8c2ec8b..d7221b9 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml @@ -40,19 +40,6 @@ <name>snappy-devel</name> </package> <package> - <name>lzo</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}</name> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}-native</name> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop_${stack_version}-libhdfs</name> </package> <package> @@ -77,19 +64,6 @@ <name>snappy-devel</name> </package> <package> - <name>liblzo2-2</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}</name> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}-native</name> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop_${stack_version}-libhdfs</name> </package> </packages> @@ -123,15 +97,6 @@ <name>libsnappy-dev</name> </package> <package> - <name>hadooplzo-${stack_version}</name> - <condition>should_install_lzo</condition> - </package> - <package> - <name>liblzo2-2</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> <name>libhdfs0-${stack_version}</name> </package> </packages> http://git-wip-us.apache.org/repos/asf/ambari/blob/7eb14a0a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml index ccf9a4e..86531cc 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml @@ -72,19 +72,6 @@ <name>snappy-devel</name> </package> <package> - <name>lzo</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}</name> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}-native</name> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop_${stack_version}-libhdfs</name> </package> <package> @@ -109,19 +96,6 @@ <name>snappy-devel</name> </package> <package> - <name>liblzo2-2</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}</name> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}-native</name> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop_${stack_version}-libhdfs</name> </package> </packages> @@ -155,10 +129,6 @@ <name>libsnappy-dev</name> </package> <package> - <name>hadooplzo-${stack_version}</name> - <condition>should_install_lzo</condition> - </package> - <package> <name>libhdfs0-${stack_version}</name> </package> </packages> http://git-wip-us.apache.org/repos/asf/ambari/blob/7eb14a0a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/metainfo.xml index 95a5f84..775508e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/metainfo.xml @@ -50,19 +50,6 @@ <name>snappy-devel</name> </package> <package> - <name>lzo</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}</name> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}-native</name> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop_${stack_version}-libhdfs</name> </package> </packages> @@ -84,19 +71,6 @@ <name>snappy-devel</name> </package> <package> - <name>liblzo2-2</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}</name> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}-native</name> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop_${stack_version}-libhdfs</name> </package> </packages> @@ -130,10 +104,6 @@ <name>libsnappy-dev</name> </package> <package> - <name>hadooplzo-${stack_version}</name> - <condition>should_install_lzo</condition> - </package> - <package> <name>libhdfs0-${stack_version}</name> </package> </packages> http://git-wip-us.apache.org/repos/asf/ambari/blob/7eb14a0a/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json b/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json index 86ca03a..29cbddc 100644 --- a/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json +++ b/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json @@ -55,7 +55,8 @@ "mysql_jdbc_url": "http://c6401.ambari.apache.org:8080/resources//mysql-connector-java.jar";, "custom_mysql_jdbc_name" : "mysql-connector-java.jar", "custom_oracle_jdbc_name" : "oracle-jdbc-driver.jar", - "custom_postgres_jdbc_name" : "test-postgres-jdbc.jar" + "custom_postgres_jdbc_name" : "test-postgres-jdbc.jar", + "gpl_license_accepted": "true" }, "commandType": "EXECUTION_COMMAND", "roleParams": {
