AMBARI-11306. RU to copy correct version of tarball, including tez.tar.gz and mapreduce.tar.gz (alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f1787236 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f1787236 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f1787236 Branch: refs/heads/trunk Commit: f1787236115ea091a15e1e3a45deb4ec9be05f85 Parents: 818dadf Author: Alejandro Fernandez <[email protected]> Authored: Wed May 20 13:11:02 2015 -0700 Committer: Alejandro Fernandez <[email protected]> Committed: Thu May 21 20:10:59 2015 -0700 ---------------------------------------------------------------------- .../libraries/functions/check_process_status.py | 5 +- .../libraries/functions/copy_tarball.py | 141 +++++++++++++++++++ .../libraries/functions/hive_check.py | 4 +- .../libraries/resources/hdfs_resource.py | 2 +- .../2.1.0.2.0/package/scripts/params_linux.py | 2 +- .../HIVE/0.12.0.2.0/package/scripts/hive.py | 129 +++++++++-------- .../0.12.0.2.0/package/scripts/hive_server.py | 36 ++--- .../0.12.0.2.0/package/scripts/hive_service.py | 10 +- .../0.12.0.2.0/package/scripts/params_linux.py | 53 +++---- .../0.12.0.2.0/package/scripts/params_linux.py | 12 +- .../0.12.0.2.0/package/scripts/service_check.py | 31 ++-- .../package/scripts/job_history_server.py | 17 +-- .../SPARK/1.2.0.2.2/package/scripts/params.py | 2 - .../1.2.0.2.2/package/scripts/service_check.py | 1 + .../1.2.0.2.2/package/scripts/spark_service.py | 16 +-- .../2.1.0.2.0/package/scripts/historyserver.py | 26 ++-- .../2.1.0.2.0/package/scripts/params_linux.py | 10 +- .../package/scripts/resourcemanager.py | 25 ++-- .../2.1.0.2.0/package/scripts/status_params.py | 9 +- .../HDP/2.2/configuration/cluster-env.xml | 83 ----------- .../2.3.GlusterFS/configuration/cluster-env.xml | 83 ----------- .../configuration/cluster-env.xml.version | 83 ----------- .../catalog/UpgradeCatalog_1.3_to_2.2.json | 14 +- .../catalog/UpgradeCatalog_2.0_to_2.2.2.json | 14 -- .../catalog/UpgradeCatalog_2.0_to_2.2.4.json | 12 -- .../catalog/UpgradeCatalog_2.1_to_2.2.2.json | 14 -- .../catalog/UpgradeCatalog_2.1_to_2.2.4.json | 12 -- .../server/upgrade/UpgradeCatalog200Test.java | 12 -- .../stacks/2.0.6/HIVE/test_hive_server.py | 77 +++------- .../stacks/2.0.6/PIG/test_pig_service_check.py | 2 + .../stacks/2.0.6/YARN/test_historyserver.py | 20 +-- .../stacks/2.0.6/YARN/test_resourcemanager.py | 4 +- .../stacks/2.0.6/configs/client-upgrade.json | 18 +-- .../python/stacks/2.0.6/configs/default.json | 26 +--- .../2.0.6/configs/default_hive_nn_ha.json | 4 +- .../2.0.6/configs/default_hive_nn_ha_2.json | 4 +- .../2.0.6/configs/default_no_install.json | 4 +- .../python/stacks/2.0.6/configs/hbase-2.2.json | 20 +-- .../stacks/2.0.6/configs/hbase-check-2.2.json | 20 +-- .../stacks/2.0.6/configs/hbase-preupgrade.json | 22 +-- .../stacks/2.0.6/configs/hbase-rs-2.2.json | 20 +-- .../python/stacks/2.0.6/configs/nn_ru_lzo.json | 24 +--- .../2.0.6/configs/ranger-namenode-start.json | 14 +- .../python/stacks/2.0.6/configs/secured.json | 13 +- .../2.0.6/configs/zk-service_check_2.2.json | 18 +-- .../stacks/2.1/configs/client-upgrade.json | 16 +-- .../stacks/2.2/PIG/test_pig_service_check.py | 30 ++-- .../stacks/2.2/SPARK/test_job_history_server.py | 44 +----- .../2.2/SPARK/test_spark_service_check.py | 2 + .../test/python/stacks/2.2/configs/default.json | 13 +- .../stacks/2.2/configs/falcon-upgrade.json | 18 +-- .../python/stacks/2.2/configs/hive-upgrade.json | 14 +- .../journalnode-upgrade-hdfs-secure.json | 26 +--- .../stacks/2.2/configs/journalnode-upgrade.json | 24 +--- .../stacks/2.2/configs/oozie-downgrade.json | 14 +- .../stacks/2.2/configs/oozie-upgrade.json | 14 +- .../2.2/configs/pig-service-check-secure.json | 26 +--- .../2.2/configs/ranger-admin-upgrade.json | 26 +--- .../2.2/configs/ranger-usersync-upgrade.json | 26 +--- .../test/python/stacks/2.2/configs/secured.json | 13 +- .../2.2/configs/spark-job-history-server.json | 24 +--- .../stacks/2.3/configs/hbase_default.json | 24 +--- .../python/stacks/2.3/configs/hbase_secure.json | 24 +--- .../stacks/2.3/configs/storm_default.json | 24 +--- .../ru_execute_tasks_namenode_prepare.json | 24 +--- 65 files changed, 500 insertions(+), 1094 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-common/src/main/python/resource_management/libraries/functions/check_process_status.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/check_process_status.py b/ambari-common/src/main/python/resource_management/libraries/functions/check_process_status.py index 21cb3bb..6f39c28 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/check_process_status.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/check_process_status.py @@ -39,12 +39,13 @@ def check_process_status(pid_file): from resource_management.core import sudo if not pid_file or not os.path.isfile(pid_file): + Logger.info("Pid file {0} is empty or does not exist".format(str(pid_file))) raise ComponentIsNotRunning() try: pid = int(sudo.read_file(pid_file)) except: - Logger.debug("Pid file {0} does not exist".format(pid_file)) + Logger.info("Pid file {0} does not exist or does not contain a process id number".format(pid_file)) raise ComponentIsNotRunning() try: @@ -55,6 +56,6 @@ def check_process_status(pid_file): # process ID or process group ID. os.kill(pid, 0) except OSError: - Logger.debug("Process with pid {0} is not running. Stale pid file" + Logger.info("Process with pid {0} is not running. Stale pid file" " at {1}".format(pid, pid_file)) raise ComponentIsNotRunning() http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py new file mode 100644 index 0000000..acb1e9f --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +__all__ = ["copy_to_hdfs", ] + +import os +import uuid + +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.functions.default import default +from resource_management.core.logger import Logger + +STACK_VERSION_PATTERN = "{{ stack_version }}" + +TARBALL_MAP = { + "HDP": { + "tez": ("/usr/hdp/%s/tez/lib/tez.tar.gz" % STACK_VERSION_PATTERN, + "/hdp/apps/%s/tez/tez.tar.gz" % STACK_VERSION_PATTERN), + + "hive": ("/usr/hdp/%s/hive/hive.tar.gz" % STACK_VERSION_PATTERN, + "/hdp/apps/%s/hive/hive.tar.gz" % STACK_VERSION_PATTERN), + + "pig": ("/usr/hdp/%s/pig/pig.tar.gz" % STACK_VERSION_PATTERN, + "/hdp/apps/%s/pig/pig.tar.gz" % STACK_VERSION_PATTERN), + + "hadoop_streaming": ("/usr/hdp/%s/hadoop/hadoop-streaming.jar" % STACK_VERSION_PATTERN, + "/hdp/apps/%s/mapreduce/hadoop-streaming.jar" % STACK_VERSION_PATTERN), + + "sqoop": ("/usr/hdp/%s/sqoop/sqoop.tar.gz" % STACK_VERSION_PATTERN, + "/hdp/apps/%s/sqoop/sqoop.tar.gz" % STACK_VERSION_PATTERN), + + "mapreduce": ("/usr/hdp/%s/hadoop/mapreduce.tar.gz" % STACK_VERSION_PATTERN, + "/hdp/apps/%s/mapreduce/mapreduce.tar.gz" % STACK_VERSION_PATTERN) + } +} + +def copy_to_hdfs(name, user_group, owner, file_mode=0444, custom_source_file=None, custom_dest_file=None, force_execute=False): + """ + :param name: Tarball name, e.g., tez, hive, pig, sqoop. + :param user_group: Group to own the directory. + :param owner: File owner + :param file_mode: File permission + :param custom_source_file: Override the source file path + :param custom_dest_file: Override the destination file path + :param force_execute: If true, will execute the HDFS commands immediately, otherwise, will defer to the calling function. + :return: Will return True if successful, otherwise, False. + """ + import params + + if params.stack_name is None or params.stack_name.upper() not in TARBALL_MAP: + Logger.error("Cannot copy %s tarball to HDFS because stack %s does not support this operation." % (str(name), str(params.stack_name))) + return -1 + + if name is None or name.lower() not in TARBALL_MAP[params.stack_name.upper()]: + Logger.warning("Cannot copy tarball to HDFS because %s is not supported in stack for this operation." % (str(name), str(params.stack_name))) + return -1 + + (source_file, dest_file) = TARBALL_MAP[params.stack_name.upper()][name.lower()] + + if custom_source_file is not None: + source_file = custom_source_file + + if custom_dest_file is not None: + dest_file = custom_dest_file + + upgrade_direction = default("/commandParams/upgrade_direction", None) + is_rolling_upgrade = upgrade_direction is not None + current_version = default("/hostLevelParams/current_version", None) + if is_rolling_upgrade: + # This is the version going to. In the case of a downgrade, it is the lower version. + current_version = default("/commandParams/version", None) + + if current_version is None: + message_suffix = " during rolling %s" % str(upgrade_direction) if is_rolling_upgrade else "" + Logger.warning("Cannot copy %s tarball because unable to determine current version%s." % (str(name), message_suffix)) + return False + + source_file = source_file.replace(STACK_VERSION_PATTERN, current_version) + dest_file = dest_file.replace(STACK_VERSION_PATTERN, current_version) + + if not os.path.exists(source_file): + Logger.warning("WARNING. Cannot copy %s tarball because file does not exist: %s . It is possible that this component is not installed on this host." % (str(name), str(source_file))) + return False + + # Because CopyFromLocal does not guarantee synchronization, it's possible for two processes to first attempt to + # copy the file to a temporary location, then process 2 fails because the temporary file was already created by + # process 1, so process 2 tries to clean up by deleting the temporary file, and then process 1 + # cannot finish the copy to the final destination, and both fail! + # For this reason, the file name on the destination must be unique, and we then rename it to the intended value. + # The rename operation is synchronized by the Namenode. + + #unique_string = str(uuid.uuid4())[:8] + #temp_dest_file = dest_file + "." + unique_string + + # The logic above cannot be used until fast-hdfs-resource.jar supports the mv command, or it switches + # to WebHDFS. + + + # If the directory already exists, it is a NO-OP + dest_dir = os.path.dirname(dest_file) + params.HdfsResource(dest_dir, + type="directory", + action="create_on_execute", + owner=owner, + mode=0555 + ) + + # If the file already exists, it is a NO-OP + params.HdfsResource(dest_file, + type="file", + action="create_on_execute", + source=source_file, + group=user_group, + owner=owner, + mode=0444 + ) + Logger.info("Will attempt to copy %s tarball from %s to DFS at %s." % (name, source_file, dest_file)) + + # For improved performance, force_execute should be False so that it is delayed and combined with other calls. + # If still want to run the command now, set force_execute to True + if force_execute: + params.HdfsResource(None, action="execute") + + return True \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py b/ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py index 6bd6599..55fd6bd 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py @@ -17,10 +17,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ''' + +import socket + from resource_management.core.exceptions import Fail from resource_management.core.resources import Execute from resource_management.libraries.functions import format -import socket def check_thrift_port_sasl(address, port, hive_auth="NOSASL", key=None, kinitcmd=None, smokeuser='ambari-qa', http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py b/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py index 92a043f..970e1e9 100644 --- a/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py +++ b/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py @@ -27,7 +27,7 @@ from resource_management.core.base import Resource, ForcedListArgument, Resource Calling a lot of hadoop commands takes too much time. The cause is that for every call new connection initialized, with datanodes, namenode. -While this resource can gather the dicteroies/files to create/delete/copyFromLocal. +While this resource can gather the directories/files to create/delete/copyFromLocal. And after just with one call create all that. action = create_delayed / delete_delayed. Are for gathering information about what you want http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py index 839d980..b29eb8e 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py @@ -224,7 +224,7 @@ if 'dfs.namenode.rpc-address' in config['configurations']['hdfs-site']: else: namenode_address = config['configurations']['core-site']['fs.defaultFS'] -fs_checkpoint_dirs = config['configurations']['hdfs-site']['dfs.namenode.checkpoint.dir'].split(',') +fs_checkpoint_dirs = default("/configurations/hdfs-site/dfs.namenode.checkpoint.dir", "").split(',') dfs_data_dir = config['configurations']['hdfs-site']['dfs.datanode.data.dir'] dfs_data_dir = ",".join([re.sub(r'^\[.+\]', '', dfs_dir.strip()) for dfs_dir in dfs_data_dir.split(",")]) http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py index 6edca7d..cf08e1c 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py @@ -18,14 +18,25 @@ limitations under the License. """ -from resource_management import * -from resource_management.libraries import functions -import sys import os import glob +from urlparse import urlparse + +from resource_management.libraries.script.script import Script +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.functions.copy_tarball import copy_to_hdfs +from resource_management.libraries.functions.version import compare_versions +from resource_management.core.resources.service import ServiceConfig +from resource_management.core.resources.system import File, Execute, Directory +from resource_management.core.source import StaticFile, Template, DownloadSource, InlineTemplate +from resource_management.core.shell import as_user +from resource_management.libraries.functions.is_empty import is_empty +from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.libraries.functions.format import format +from resource_management.core.exceptions import Fail + from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from ambari_commons import OSConst -from urlparse import urlparse @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) @@ -81,24 +92,16 @@ def hive(name=None): import params if name == 'hiveserver2': - - if params.hdp_stack_version_major != "" and compare_versions(params.hdp_stack_version_major, '2.2') >=0: - params.HdfsResource(InlineTemplate(params.mapreduce_tar_destination).get_content(), - type="file", - action="create_on_execute", - source=params.mapreduce_tar_source, - group=params.user_group, - mode=params.tarballs_mode - ) - + # HDP 2.1.* or lower if params.hdp_stack_version_major != "" and compare_versions(params.hdp_stack_version_major, "2.2.0.0") < 0: params.HdfsResource(params.webhcat_apps_dir, - type="directory", - action="create_on_execute", - owner=params.webhcat_user, - mode=0755 - ) - + type="directory", + action="create_on_execute", + owner=params.webhcat_user, + mode=0755 + ) + + # Create webhcat dirs. if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir: params.HdfsResource(params.hcat_hdfs_user_dir, type="directory", @@ -106,56 +109,66 @@ def hive(name=None): owner=params.hcat_user, mode=params.hcat_hdfs_user_mode ) + params.HdfsResource(params.webhcat_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.webhcat_user, mode=params.webhcat_hdfs_user_mode ) - - for src_filepath in glob.glob(params.hadoop_streaming_tar_source): - src_filename = os.path.basename(src_filepath) - params.HdfsResource(InlineTemplate(params.hadoop_streaming_tar_destination_dir).get_content() + '/' + src_filename, - type="file", - action="create_on_execute", - source=src_filepath, - group=params.user_group, - mode=params.tarballs_mode - ) - - if (os.path.isfile(params.pig_tar_source)): - params.HdfsResource(InlineTemplate(params.pig_tar_destination).get_content(), - type="file", - action="create_on_execute", - source=params.pig_tar_source, - group=params.user_group, - mode=params.tarballs_mode - ) - - params.HdfsResource(InlineTemplate(params.hive_tar_destination).get_content(), - type="file", - action="create_on_execute", - source=params.hive_tar_source, - group=params.user_group, - mode=params.tarballs_mode - ) - - for src_filepath in glob.glob(params.sqoop_tar_source): - src_filename = os.path.basename(src_filepath) - params.HdfsResource(InlineTemplate(params.sqoop_tar_destination_dir).get_content() + '/' + src_filename, - type="file", - action="create_on_execute", - source=src_filepath, - group=params.user_group, - mode=params.tarballs_mode - ) - + + # ****** Begin Copy Tarballs ****** + # ********************************* + # HDP 2.2 or higher, copy mapreduce.tar.gz to HDFS + if params.hdp_stack_version_major != "" and compare_versions(params.hdp_stack_version_major, '2.2') >= 0: + copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user) + + # Always copy pig.tar.gz and hive.tar.gz using the appropriate mode. + # This can use a different source and dest location to account for both HDP 2.1 and 2.2 + copy_to_hdfs("pig", + params.user_group, + params.hdfs_user, + file_mode=params.tarballs_mode, + custom_source_file=params.pig_tar_source, + custom_dest_file=params.pig_tar_dest_file) + copy_to_hdfs("hive", + params.user_group, + params.hdfs_user, + file_mode=params.tarballs_mode, + custom_source_file=params.hive_tar_source, + custom_dest_file=params.hive_tar_dest_file) + + wildcard_tarballs = ["sqoop", "hadoop_streaming"] + for tarball_name in wildcard_tarballs: + source_file_pattern = eval("params." + tarball_name + "_tar_source") + dest_dir = eval("params." + tarball_name + "_tar_dest_dir") + + if source_file_pattern is None or dest_dir is None: + continue + + source_files = glob.glob(source_file_pattern) if "*" in source_file_pattern else [source_file_pattern] + for source_file in source_files: + src_filename = os.path.basename(source_file) + dest_file = os.path.join(dest_dir, src_filename) + + copy_to_hdfs(tarball_name, + params.user_group, + params.hdfs_user, + file_mode=params.tarballs_mode, + custom_source_file=source_file, + custom_dest_file=dest_file) + # ******* End Copy Tarballs ******* + # ********************************* + + # Create Hive Metastore Warehouse Dir params.HdfsResource(params.hive_apps_whs_dir, type="directory", action="create_on_execute", owner=params.hive_user, mode=0777 ) + + # Create Hive User Dir params.HdfsResource(params.hive_hdfs_user_dir, type="directory", action="create_on_execute", http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py index aeac939..0d13bea 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py @@ -17,20 +17,30 @@ See the License for the specific language governing permissions and limitations under the License. """ -import hive_server_upgrade -from resource_management import * -from hive import hive -from hive_service import hive_service + +from resource_management.libraries.script.script import Script +from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import hdp_select +from resource_management.libraries.functions import format +from resource_management.libraries.functions.copy_tarball import copy_to_hdfs from resource_management.libraries.functions.get_hdp_version import get_hdp_version +from resource_management.libraries.functions.check_process_status import check_process_status +from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version from resource_management.libraries.functions.security_commons import build_expectations, \ cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ FILE_TYPE_XML +from ambari_commons import OSCheck, OSConst +if OSCheck.is_windows_family(): + from resource_management.libraries.functions.windows_service_utils import check_windows_service_status from setup_ranger_hive import setup_ranger_hive from ambari_commons.os_family_impl import OsFamilyImpl -from ambari_commons import OSConst +from resource_management.core.logger import Logger + +import hive_server_upgrade +from hive import hive +from hive_service import hive_service class HiveServer(Script): @@ -100,18 +110,12 @@ class HiveServerDefault(HiveServer): if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: conf_select.select(params.stack_name, "hive", params.version) hdp_select.select("hive-server2", params.version) - old = params.hdp_stack_version - try: - params.hdp_stack_version = get_hdp_version('hive-server2') - params.HdfsResource(InlineTemplate(params.mapreduce_tar_destination).get_content(), - type="file", - action="create_on_execute", - source=params.mapreduce_tar_source, - group=params.user_group, - mode=params.tarballs_mode) + + # Copy mapreduce.tar.gz and tez.tar.gz to HDFS + resource_created = copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user) + resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user) or resource_created + if resource_created: params.HdfsResource(None, action="execute") - finally: - params.hdp_stack_version = old def security_status(self, env): import status_params http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py index ea70b8a..43378a9 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py @@ -18,11 +18,17 @@ limitations under the License. """ -from resource_management import * -import sys import os import time + from resource_management.core import shell +from resource_management.libraries.functions.format import format +from resource_management.core.resources.system import File, Execute +from resource_management.core.resources.service import Service +from resource_management.core.exceptions import Fail +from resource_management.core.shell import as_user +from resource_management.libraries.functions.hive_check import check_thrift_port_sasl + from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from ambari_commons import OSConst http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index 934aadf..647892a 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -29,6 +29,7 @@ from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.functions.default import default from resource_management.libraries.functions.format import format from resource_management.libraries.functions.version import format_hdp_stack_version +from resource_management.libraries.functions.copy_tarball import STACK_VERSION_PATTERN from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.script.script import Script from resource_management.libraries.functions.get_port_from_url import get_port_from_url @@ -50,7 +51,7 @@ stack_version_unformatted = str(config['hostLevelParams']['stack_version']) hdp_stack_version_major = format_hdp_stack_version(stack_version_unformatted) stack_is_hdp21 = Script.is_hdp_stack_greater_or_equal("2.0") and Script.is_hdp_stack_less_than("2.2") -# this is not avaliable on INSTALL action because hdp-select is not available +# this is not available on INSTALL action because hdp-select is not available hdp_stack_version = functions.get_hdp_version('hive-server2') # New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade @@ -58,13 +59,16 @@ version = default("/commandParams/version", None) hadoop_bin_dir = "/usr/bin" hadoop_home = '/usr' -hadoop_streeming_jars = '/usr/lib/hadoop-mapreduce/hadoop-streaming-*.jar' hive_bin = '/usr/lib/hive/bin' hive_lib = '/usr/lib/hive/lib/' hive_var_lib = '/var/lib/hive' + +# These tar folders were used in HDP 2.1 +hadoop_streaming_jars = '/usr/lib/hadoop-mapreduce/hadoop-streaming-*.jar' pig_tar_file = '/usr/share/HDP-webhcat/pig.tar.gz' hive_tar_file = '/usr/share/HDP-webhcat/hive.tar.gz' sqoop_tar_file = '/usr/share/HDP-webhcat/sqoop*.tar.gz' + hive_specific_configs_supported = False hive_etc_dir_prefix = "/etc/hive" limits_conf_dir = "/etc/security/limits.d" @@ -102,41 +106,38 @@ if Script.is_hdp_stack_greater_or_equal("2.2"): webhcat_bin_dir = '/usr/hdp/current/hive-webhcat/sbin' # --- Tarballs --- + # DON'T CHANGE THESE VARIABLE NAMES + # Values don't change from those in copy_tarball.py + hive_tar_source = "/usr/hdp/%s/hive/hive.tar.gz" % STACK_VERSION_PATTERN + pig_tar_source = "/usr/hdp/%s/pig/pig.tar.gz" % STACK_VERSION_PATTERN + hive_tar_dest_file = "/hdp/apps/%s/hive/hive.tar.gz" % STACK_VERSION_PATTERN + pig_tar_dest_file = "/hdp/apps/%s/pig/pig.tar.gz" % STACK_VERSION_PATTERN - hive_tar_source = config['configurations']['cluster-env']['hive_tar_source'] - pig_tar_source = config['configurations']['cluster-env']['pig_tar_source'] - hadoop_streaming_tar_source = config['configurations']['cluster-env']['hadoop-streaming_tar_source'] - sqoop_tar_source = config['configurations']['cluster-env']['sqoop_tar_source'] - mapreduce_tar_source = config['configurations']['cluster-env']['mapreduce_tar_source'] - tez_tar_source = config['configurations']['cluster-env']['tez_tar_source'] - - hive_tar_destination = config['configurations']['cluster-env']['hive_tar_destination_folder'] + "/" + os.path.basename(hive_tar_source) - pig_tar_destination = config['configurations']['cluster-env']['pig_tar_destination_folder'] + "/" + os.path.basename(pig_tar_source) - hadoop_streaming_tar_destination_dir = config['configurations']['cluster-env']['hadoop-streaming_tar_destination_folder'] - sqoop_tar_destination_dir = config['configurations']['cluster-env']['sqoop_tar_destination_folder'] - mapreduce_tar_destination = config['configurations']['cluster-env']['mapreduce_tar_destination_folder'] + "/" + os.path.basename(mapreduce_tar_source) - tez_tar_destination = config['configurations']['cluster-env']['tez_tar_destination_folder'] + "/" + os.path.basename(tez_tar_source) + + hadoop_streaming_tar_source = "/usr/hdp/%s/hadoop/hadoop-streaming.jar" % STACK_VERSION_PATTERN + sqoop_tar_source = "/usr/hdp/%s/sqoop/sqoop.tar.gz" % STACK_VERSION_PATTERN + hadoop_streaming_tar_dest_dir = "/hdp/apps/%s/mapreduce/" % STACK_VERSION_PATTERN + sqoop_tar_dest_dir = "/hdp/apps/%s/sqoop/" % STACK_VERSION_PATTERN tarballs_mode = 0444 else: # --- Tarballs --- + webhcat_apps_dir = "/apps/webhcat" + + # In HDP 2.1, the tarballs were copied from and to different locations. + # DON'T CHANGE THESE VARIABLE NAMES hive_tar_source = hive_tar_file pig_tar_source = pig_tar_file - hadoop_streaming_tar_source = hadoop_streeming_jars - sqoop_tar_source = sqoop_tar_file + hive_tar_dest_file = webhcat_apps_dir + "/hive.tar.gz" + pig_tar_dest_file = webhcat_apps_dir + "/pig.tar.gz" - webhcat_apps_dir = "/apps/webhcat" - - hive_tar_destination = webhcat_apps_dir + "/" + os.path.basename(hive_tar_source) - pig_tar_destination = webhcat_apps_dir + "/" + os.path.basename(pig_tar_source) - hadoop_streaming_tar_destination_dir = webhcat_apps_dir - sqoop_tar_destination_dir = webhcat_apps_dir + hadoop_streaming_tar_source = hadoop_streaming_jars # this contains * + sqoop_tar_source = sqoop_tar_file # this contains * + hadoop_streaming_tar_dest_dir = webhcat_apps_dir + sqoop_tar_dest_dir = webhcat_apps_dir tarballs_mode = 0755 - - - execute_path = os.environ['PATH'] + os.pathsep + hive_bin + os.pathsep + hadoop_bin_dir hive_metastore_user_name = config['configurations']['hive-site']['javax.jdo.option.ConnectionUserName'] hive_jdbc_connection_url = config['configurations']['hive-site']['javax.jdo.option.ConnectionURL'] http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_linux.py index fc39bc6..d7ff0ec 100644 --- a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_linux.py @@ -18,15 +18,13 @@ limitations under the License. Ambari Agent """ -from resource_management import * -from resource_management.libraries.functions import format + +from resource_management.libraries.script.script import Script +from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.functions import conf_select from resource_management.libraries.functions.version import format_hdp_stack_version from resource_management.libraries.functions.default import default from resource_management.libraries.functions import get_kinit_path -from resource_management.libraries.script.script import Script - -import os # server configurations config = Script.get_config() @@ -52,10 +50,6 @@ if Script.is_hdp_stack_greater_or_equal("2.2"): pig_conf_dir = "/usr/hdp/current/pig-client/conf" hadoop_home = '/usr/hdp/current/hadoop-client' pig_bin_dir = '/usr/hdp/current/pig-client/bin' - - tez_tar_source = config['configurations']['cluster-env']['tez_tar_source'] - tez_tar_destination = config['configurations']['cluster-env']['tez_tar_destination_folder'] + "/" + os.path.basename(tez_tar_source) - hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py index 5f33b1d..b090a04 100644 --- a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py +++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py @@ -18,11 +18,19 @@ limitations under the License. Ambari Agent """ - -from resource_management import * -from resource_management.libraries import functions +import os + +from resource_management.libraries.script.script import Script +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.resources.execute_hadoop import ExecuteHadoop +from resource_management.libraries.functions.version import compare_versions +from resource_management.libraries.functions.copy_tarball import copy_to_hdfs +from resource_management.libraries.functions import format from ambari_commons import OSConst from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from resource_management.core.resources.system import File, Execute +from resource_management.core.source import StaticFile + class PigServiceCheck(Script): pass @@ -61,7 +69,8 @@ class PigServiceCheckLinux(PigServiceCheck): tries = 3, try_sleep = 5, path = format('{pig_bin_dir}:/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'), - user = params.smokeuser + user = params.smokeuser, + logoutput = True ) test_cmd = format("fs -test -e {output_dir}") @@ -86,14 +95,9 @@ class PigServiceCheckLinux(PigServiceCheck): ) # Check for Pig-on-Tez - params.HdfsResource(InlineTemplate(params.tez_tar_destination).get_content(), - type="file", - action="create_on_execute", - source=params.tez_tar_source, - group=params.user_group, - owner=params.hdfs_user - ) - params.HdfsResource(None, action="execute") + resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user) + if resource_created: + params.HdfsResource(None, action="execute") if params.security_enabled: kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};") @@ -105,7 +109,8 @@ class PigServiceCheckLinux(PigServiceCheck): tries = 3, try_sleep = 5, path = format('{pig_bin_dir}:/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'), - user = params.smokeuser + user = params.smokeuser, + logoutput = True ) ExecuteHadoop(test_cmd, http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py index c96be5d..106b8cc 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py @@ -20,13 +20,13 @@ limitations under the License. import sys import os + +from resource_management.libraries.script.script import Script from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import hdp_select from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version -from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.copy_tarball import copy_to_hdfs from resource_management.libraries.functions.check_process_status import check_process_status -from resource_management.core.resources import Execute -from resource_management.core.exceptions import ComponentIsNotRunning from resource_management.core.logger import Logger from resource_management.core import shell from setup_spark import * @@ -78,14 +78,9 @@ class JobHistoryServer(Script): conf_select.select(params.stack_name, "spark", params.version) hdp_select.select("spark-historyserver", params.version) - params.HdfsResource(InlineTemplate(params.tez_tar_destination).get_content(), - type="file", - action="create_on_execute", - source=params.tez_tar_source, - group=params.user_group, - owner=params.hdfs_user - ) - params.HdfsResource(None, action="execute") + resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user) + if resource_created: + params.HdfsResource(None, action="execute") if __name__ == "__main__": JobHistoryServer().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py index 34d1cb8..348f49d 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py @@ -70,8 +70,6 @@ if Script.is_hdp_stack_greater_or_equal("2.2"): spark_log_dir = config['configurations']['spark-env']['spark_log_dir'] spark_pid_dir = status_params.spark_pid_dir spark_home = format("/usr/hdp/current/{component_directory}") - tez_tar_source = config['configurations']['cluster-env']['tez_tar_source'] - tez_tar_destination = config['configurations']['cluster-env']['tez_tar_destination_folder'] + "/" + os.path.basename(tez_tar_source) java_home = config['hostLevelParams']['java_home'] http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/service_check.py index f5b0167..b28782b 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/service_check.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/service_check.py @@ -36,6 +36,7 @@ class SparkServiceCheck(Script): Execute(format("curl -s -o /dev/null -w'%{{http_code}}' --negotiate -u: -k http://{spark_history_server_host}:{spark_history_ui_port} | grep 200"), tries = 10, try_sleep=3, + logoutput=True ) if __name__ == "__main__": http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py index 840b81d..834e208 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py @@ -17,7 +17,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ''' -from resource_management import * + +from resource_management.libraries.script.script import Script +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.functions.copy_tarball import copy_to_hdfs +from resource_management.libraries.functions import format +from resource_management.core.resources.system import File, Execute def spark_service(action): import params @@ -27,13 +32,8 @@ def spark_service(action): spark_kinit_cmd = format("{kinit_path_local} -kt {spark_kerberos_keytab} {spark_principal}; ") Execute(spark_kinit_cmd, user=params.spark_user) - params.HdfsResource(InlineTemplate(params.tez_tar_destination).get_content(), - type="file", - action="create_on_execute", - source=params.tez_tar_source, - group=params.user_group, - owner=params.hdfs_user - ) + resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user) + if resource_created: params.HdfsResource(None, action="execute") no_op_test = format( http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py index 6b56511..af37153 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py @@ -19,14 +19,20 @@ Ambari Agent """ -from resource_management import * +from resource_management.libraries.script.script import Script +from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import hdp_select +from resource_management.libraries.functions.check_process_status import check_process_status +from resource_management.libraries.functions.copy_tarball import copy_to_hdfs from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version from resource_management.libraries.functions.format import format from resource_management.libraries.functions.security_commons import build_expectations, \ cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ FILE_TYPE_XML +from resource_management.core.source import Template +from resource_management.core.logger import Logger + from yarn import yarn from service import service from ambari_commons import OSConst @@ -73,14 +79,7 @@ class HistoryServerDefault(HistoryServer): if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: conf_select.select(params.stack_name, "hadoop", params.version) hdp_select.select("hadoop-mapreduce-historyserver", params.version) - params.HdfsResource(InlineTemplate(params.mapreduce_tar_destination).get_content(), - type="file", - action="create_on_execute", - source=params.mapreduce_tar_source, - owner=params.hdfs_user, - group=params.user_group, - mode=0444, - ) + copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user) params.HdfsResource(None, action="execute") @@ -90,14 +89,7 @@ class HistoryServerDefault(HistoryServer): self.configure(env) # FOR SECURITY if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: - params.HdfsResource(InlineTemplate(params.mapreduce_tar_destination).get_content(), - type="file", - action="create_on_execute", - source=params.mapreduce_tar_source, - owner=params.hdfs_user, - group=params.user_group, - mode=0444, - ) + copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user) params.HdfsResource(None, action="execute") service('historyserver', action='start', serviceName='mapreduce') http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index 90b23de..2198bcc 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -20,13 +20,13 @@ Ambari Agent """ import os +from resource_management.libraries.script.script import Script +from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.functions import conf_select -from resource_management import * from resource_management.libraries.functions import format from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.functions.version import format_hdp_stack_version from resource_management.libraries.functions.default import default -from resource_management.libraries.script.script import Script from resource_management.libraries import functions import status_params @@ -91,16 +91,10 @@ if Script.is_hdp_stack_greater_or_equal("2.2"): hadoop_yarn_home = format("/usr/hdp/current/{yarn_role_root}") yarn_bin = format("/usr/hdp/current/{yarn_role_root}/sbin") yarn_container_bin = format("/usr/hdp/current/{yarn_role_root}/bin") - - mapreduce_tar_source = config['configurations']['cluster-env']['mapreduce_tar_source'] - mapreduce_tar_destination = config['configurations']['cluster-env']['mapreduce_tar_destination_folder'] + "/" + os.path.basename(mapreduce_tar_source) # the configuration direction for HDFS/YARN/MapR is the hadoop config # directory, which is symlinked by hadoop-client only hadoop_conf_dir = "/usr/hdp/current/hadoop-client/conf" - tez_tar_source = config['configurations']['cluster-env']['tez_tar_source'] - tez_tar_destination = config['configurations']['cluster-env']['tez_tar_destination_folder'] + "/" + os.path.basename(tez_tar_source) - limits_conf_dir = "/etc/security/limits.d" execute_path = os.environ['PATH'] + os.pathsep + hadoop_bin_dir + os.pathsep + yarn_container_bin http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py index b0393a0..456d7bf 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py @@ -19,20 +19,28 @@ Ambari Agent """ -from resource_management import * +from resource_management.libraries.script.script import Script +from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import hdp_select +from resource_management.libraries.functions.check_process_status import check_process_status +from resource_management.libraries.functions.copy_tarball import copy_to_hdfs from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version from resource_management.libraries.functions.security_commons import build_expectations, \ cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ FILE_TYPE_XML +from resource_management.core.resources.system import File, Execute +from resource_management.core.source import Template +from resource_management.core.logger import Logger + + + from install_jars import install_tez_jars from yarn import yarn from service import service from ambari_commons import OSConst from ambari_commons.os_family_impl import OsFamilyImpl from setup_ranger_yarn import setup_ranger_yarn -import os class Resourcemanager(Script): @@ -108,16 +116,9 @@ class ResourcemanagerDefault(Resourcemanager): if not Script.is_hdp_stack_greater_or_equal("2.2"): install_tez_jars() else: - # will work only for stack versions >=2.2 - if os.path.exists(params.tez_tar_source): - params.HdfsResource(InlineTemplate(params.tez_tar_destination).get_content(), - type="file", - action="create_on_execute", - source=params.tez_tar_source, - group=params.user_group, - owner=params.hdfs_user - ) - params.HdfsResource(None, action="execute") + resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user) + if resource_created: + params.HdfsResource(None, action="execute") service('resourcemanager', action='start') def status(self, env): http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/status_params.py index ebef301..b4c5b6e 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/status_params.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/status_params.py @@ -17,9 +17,10 @@ See the License for the specific language governing permissions and limitations under the License. """ - -from resource_management import * -from resource_management.libraries.functions import conf_select +from resource_management.libraries.script.script import Script +from resource_management.libraries import functions +from resource_management.libraries.functions import format +from resource_management.libraries.functions.default import default from ambari_commons import OSCheck config = Script.get_config() @@ -51,7 +52,7 @@ else: yarn_historyserver_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-timelineserver.pid") # *-historyserver.pid is deprecated mapred_historyserver_pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-historyserver.pid") - hadoop_conf_dir = conf_select.get_hadoop_conf_dir() + hadoop_conf_dir = functions.conf_select.get_hadoop_conf_dir() hostname = config['hostname'] kinit_path_local = functions.get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-server/src/main/resources/stacks/HDP/2.2/configuration/cluster-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/configuration/cluster-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/configuration/cluster-env.xml index 281b821..f1fa4de 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/configuration/cluster-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/configuration/cluster-env.xml @@ -21,87 +21,4 @@ --> <configuration> - - <!-- The properties that end in tar_source describe the pattern of where the tar.gz files come from. - They will replace {{ hdp_stack_version }} with the "#.#.#.#" value followed by -* (which is the build number in HDP 2.2). - When copying those tarballs, Ambari will look up the corresponding tar_destination_folder property to know where it - should be copied to. - All of the destination folders must begin with hdfs:// - Please note that the spaces inside of {{ ... }} are important. - - IMPORTANT: Any properties included here must also be declared in site_properties.js - - --> - <!-- Tez tarball is needed by Hive Server when using the Tez execution engine. --> - <property> - <name>tez_tar_source</name> - <value>/usr/hdp/current/tez-client/lib/tez.tar.gz</value> - <description>Source file path that uses dynamic variables and regex to copy the file to HDFS.</description> - </property> - <property> - <name>tez_tar_destination_folder</name> - <value>hdfs:///hdp/apps/{{ hdp_stack_version }}/tez/</value> - <description>Destination HDFS folder for the file.</description> - </property> - - <!-- Hive tarball is needed by WebHCat. --> - <property> - <name>hive_tar_source</name> - <value>/usr/hdp/current/hive-client/hive.tar.gz</value> - <description>Source file path that uses dynamic variables and regex to copy the file to HDFS.</description> - </property> - <property> - <name>hive_tar_destination_folder</name> - <value>hdfs:///hdp/apps/{{ hdp_stack_version }}/hive/</value> - <description>Destination HDFS folder for the file.</description> - </property> - - <!-- Pig tarball is needed by WebHCat. --> - <property> - <name>pig_tar_source</name> - <value>/usr/hdp/current/pig-client/pig.tar.gz</value> - <description>Source file path that uses dynamic variables and regex to copy the file to HDFS.</description> - </property> - <property> - <name>pig_tar_destination_folder</name> - <value>hdfs:///hdp/apps/{{ hdp_stack_version }}/pig/</value> - <description>Destination HDFS folder for the file.</description> - </property> - - <!-- Hadoop Streaming jar is needed by WebHCat. --> - <property> - <name>hadoop-streaming_tar_source</name> - <value>/usr/hdp/current/hadoop-mapreduce-client/hadoop-streaming.jar</value> - <description>Source file path that uses dynamic variables and regex to copy the file to HDFS.</description> - </property> - <property> - <name>hadoop-streaming_tar_destination_folder</name> - <value>hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/</value> - <description>Destination HDFS folder for the file.</description> - </property> - - <!-- Sqoop tarball is needed by WebHCat. --> - <property> - <name>sqoop_tar_source</name> - <value>/usr/hdp/current/sqoop-client/sqoop.tar.gz</value> - <description>Source file path that uses dynamic variables and regex to copy the file to HDFS.</description> - </property> - <property> - <name>sqoop_tar_destination_folder</name> - <value>hdfs:///hdp/apps/{{ hdp_stack_version }}/sqoop/</value> - <description>Destination HDFS folder for the file.</description> - </property> - - <!-- MapReduce2 tarball --> - <property> - <name>mapreduce_tar_source</name> - <value>/usr/hdp/current/hadoop-client/mapreduce.tar.gz</value> - <description>Source file path that uses dynamic variables and regex to copy the file to HDFS.</description> - </property> - <property> - <name>mapreduce_tar_destination_folder</name> - <value>hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/</value> - <description>Destination HDFS folder for the file.</description> - </property> - </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/configuration/cluster-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/configuration/cluster-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/configuration/cluster-env.xml index 485125c..f1fa4de 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/configuration/cluster-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/configuration/cluster-env.xml @@ -21,87 +21,4 @@ --> <configuration> - - <!-- The properties that end in tar_source describe the pattern of where the tar.gz files come from. - They will replace {{ hdp_stack_version }} with the "#.#.#.#" value followed by -* (which is the build number in HDP 2.2). - When copying those tarballs, Ambari will look up the corresponding tar_destination_folder property to know where it - should be copied to. - All of the destination folders must begin with hdfs:// - Please note that the spaces inside of {{ ... }} are important. - - IMPORTANT: Any properties included here must also be declared in site_properties.js - - --> - <!-- Tez tarball is needed by Hive Server when using the Tez execution engine. --> - <property> - <name>tez_tar_source</name> - <value>/usr/hdp/current/tez-client/lib/tez.tar.gz</value> - <description>Source file path that uses dynamic variables and regex to copy the file to HDFS.</description> - </property> - <property> - <name>tez_tar_destination_folder</name> - <value>glusterfs:///apps/{{ hdp_stack_version }}/tez/</value> - <description>Destination HDFS folder for the file.</description> - </property> - - <!-- Hive tarball is needed by WebHCat. --> - <property> - <name>hive_tar_source</name> - <value>/usr/hdp/current/hive-client/hive.tar.gz</value> - <description>Source file path that uses dynamic variables and regex to copy the file to HDFS.</description> - </property> - <property> - <name>hive_tar_destination_folder</name> - <value>glusterfs:///apps/{{ hdp_stack_version }}/hive/</value> - <description>Destination HDFS folder for the file.</description> - </property> - - <!-- Pig tarball is needed by WebHCat. --> - <property> - <name>pig_tar_source</name> - <value>/usr/hdp/current/pig-client/pig.tar.gz</value> - <description>Source file path that uses dynamic variables and regex to copy the file to HDFS.</description> - </property> - <property> - <name>pig_tar_destination_folder</name> - <value>glusterfs:///apps/{{ hdp_stack_version }}/pig/</value> - <description>Destination HDFS folder for the file.</description> - </property> - - <!-- Hadoop Streaming jar is needed by WebHCat. --> - <property> - <name>hadoop-streaming_tar_source</name> - <value>/usr/hdp/current/hadoop-mapreduce-client/hadoop-streaming.jar</value> - <description>Source file path that uses dynamic variables and regex to copy the file to HDFS.</description> - </property> - <property> - <name>hadoop-streaming_tar_destination_folder</name> - <value>glusterfs:///apps/{{ hdp_stack_version }}/mapreduce/</value> - <description>Destination HDFS folder for the file.</description> - </property> - - <!-- Sqoop tarball is needed by WebHCat. --> - <property> - <name>sqoop_tar_source</name> - <value>/usr/hdp/current/sqoop-client/sqoop.tar.gz</value> - <description>Source file path that uses dynamic variables and regex to copy the file to HDFS.</description> - </property> - <property> - <name>sqoop_tar_destination_folder</name> - <value>glusterfs:///apps/{{ hdp_stack_version }}/sqoop/</value> - <description>Destination HDFS folder for the file.</description> - </property> - - <!-- MapReduce2 tarball --> - <property> - <name>mapreduce_tar_source</name> - <value>/usr/hdp/current/hadoop-client/mapreduce.tar.gz</value> - <description>Source file path that uses dynamic variables and regex to copy the file to HDFS.</description> - </property> - <property> - <name>mapreduce_tar_destination_folder</name> - <value>glusterfs:///apps/{{ hdp_stack_version }}/mapreduce/</value> - <description>Destination HDFS folder for the file.</description> - </property> - </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/configuration/cluster-env.xml.version ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/configuration/cluster-env.xml.version b/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/configuration/cluster-env.xml.version index 485125c..f1fa4de 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/configuration/cluster-env.xml.version +++ b/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/configuration/cluster-env.xml.version @@ -21,87 +21,4 @@ --> <configuration> - - <!-- The properties that end in tar_source describe the pattern of where the tar.gz files come from. - They will replace {{ hdp_stack_version }} with the "#.#.#.#" value followed by -* (which is the build number in HDP 2.2). - When copying those tarballs, Ambari will look up the corresponding tar_destination_folder property to know where it - should be copied to. - All of the destination folders must begin with hdfs:// - Please note that the spaces inside of {{ ... }} are important. - - IMPORTANT: Any properties included here must also be declared in site_properties.js - - --> - <!-- Tez tarball is needed by Hive Server when using the Tez execution engine. --> - <property> - <name>tez_tar_source</name> - <value>/usr/hdp/current/tez-client/lib/tez.tar.gz</value> - <description>Source file path that uses dynamic variables and regex to copy the file to HDFS.</description> - </property> - <property> - <name>tez_tar_destination_folder</name> - <value>glusterfs:///apps/{{ hdp_stack_version }}/tez/</value> - <description>Destination HDFS folder for the file.</description> - </property> - - <!-- Hive tarball is needed by WebHCat. --> - <property> - <name>hive_tar_source</name> - <value>/usr/hdp/current/hive-client/hive.tar.gz</value> - <description>Source file path that uses dynamic variables and regex to copy the file to HDFS.</description> - </property> - <property> - <name>hive_tar_destination_folder</name> - <value>glusterfs:///apps/{{ hdp_stack_version }}/hive/</value> - <description>Destination HDFS folder for the file.</description> - </property> - - <!-- Pig tarball is needed by WebHCat. --> - <property> - <name>pig_tar_source</name> - <value>/usr/hdp/current/pig-client/pig.tar.gz</value> - <description>Source file path that uses dynamic variables and regex to copy the file to HDFS.</description> - </property> - <property> - <name>pig_tar_destination_folder</name> - <value>glusterfs:///apps/{{ hdp_stack_version }}/pig/</value> - <description>Destination HDFS folder for the file.</description> - </property> - - <!-- Hadoop Streaming jar is needed by WebHCat. --> - <property> - <name>hadoop-streaming_tar_source</name> - <value>/usr/hdp/current/hadoop-mapreduce-client/hadoop-streaming.jar</value> - <description>Source file path that uses dynamic variables and regex to copy the file to HDFS.</description> - </property> - <property> - <name>hadoop-streaming_tar_destination_folder</name> - <value>glusterfs:///apps/{{ hdp_stack_version }}/mapreduce/</value> - <description>Destination HDFS folder for the file.</description> - </property> - - <!-- Sqoop tarball is needed by WebHCat. --> - <property> - <name>sqoop_tar_source</name> - <value>/usr/hdp/current/sqoop-client/sqoop.tar.gz</value> - <description>Source file path that uses dynamic variables and regex to copy the file to HDFS.</description> - </property> - <property> - <name>sqoop_tar_destination_folder</name> - <value>glusterfs:///apps/{{ hdp_stack_version }}/sqoop/</value> - <description>Destination HDFS folder for the file.</description> - </property> - - <!-- MapReduce2 tarball --> - <property> - <name>mapreduce_tar_source</name> - <value>/usr/hdp/current/hadoop-client/mapreduce.tar.gz</value> - <description>Source file path that uses dynamic variables and regex to copy the file to HDFS.</description> - </property> - <property> - <name>mapreduce_tar_destination_folder</name> - <value>glusterfs:///apps/{{ hdp_stack_version }}/mapreduce/</value> - <description>Destination HDFS folder for the file.</description> - </property> - </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_1.3_to_2.2.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_1.3_to_2.2.json b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_1.3_to_2.2.json index f1d08b9..b95435c 100644 --- a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_1.3_to_2.2.json +++ b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_1.3_to_2.2.json @@ -148,19 +148,7 @@ "content": "\n# Set Hadoop-specific environment variables here.\n\n#Set path to where bin/hadoop is available\n#Set path to where bin/hadoop is available\nexport HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\n#set the path to where bin/hbase is available\nexport HBASE_HOME=${HBASE_HOME:-{{hbase_home}}}\n\n#Set the path to where bin/hive is available\nexport HIVE_HOME=${HIVE_HOME:-{{hive_home}}}\n\n#Set the path for where zookeper config dir is\nexport ZOOCFGDIR=${ZOOCFGDIR:-/etc/zookeeper/conf}\n\n# add libthrift in hive to sqoop class path first so hive imports work\nexport SQOOP_USER_CLASSPATH=\"`ls ${HIVE_HOME}/lib/libthrift-*.jar 2> /dev/null`:${SQOOP_USER_CLASSPATH}\"" }, "cluster-env": { - "hive_tar_source": "/usr/hdp/current/hive-client/hive.tar.gz", - "hadoop-streaming_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/", - "pig_tar_source": "/usr/hdp/current/pig-client/pig.tar.gz", - "ignore_groupsusers_create": "false", - "hadoop-streaming_tar_source": "/usr/hdp/current/hadoop-mapreduce-client/hadoop-streaming.jar", - "hive_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/hive/", - "mapreduce_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/", - "tez_tar_source": "/usr/hdp/current/tez-client/lib/tez.tar.gz", - "pig_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/pig/", - "tez_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/tez/", - "mapreduce_tar_source": "/usr/hdp/current/hadoop-client/mapreduce.tar.gz", - "sqoop_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/sqoop/", - "sqoop_tar_source": "/usr/hdp/current/sqoop-client/sqoop.tar.gz" + "ignore_groupsusers_create": "false" }, "capacity-scheduler": { "yarn.scheduler.capacity.maximum-am-resource-percent": "0.2", http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.0_to_2.2.2.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.0_to_2.2.2.json b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.0_to_2.2.2.json index 86521d1..cc4400a 100644 --- a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.0_to_2.2.2.json +++ b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.0_to_2.2.2.json @@ -65,20 +65,6 @@ "yarn.scheduler.capacity.root.accessible-node-labels.default.maximum-capacity": "-1", "yarn.scheduler.capacity.root.default-node-label-expression": " " }, - "cluster-env": { - "hadoop-streaming_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/", - "hadoop-streaming_tar_source": "/usr/hdp/current/hadoop-mapreduce-client/hadoop-streaming.jar", - "hive_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/hive/", - "hive_tar_source": "/usr/hdp/current/hive-client/hive.tar.gz", - "mapreduce_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/", - "mapreduce_tar_source": "/usr/hdp/current/hadoop-client/mapreduce.tar.gz", - "pig_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/pig/", - "pig_tar_source": "/usr/hdp/current/pig-client/pig.tar.gz", - "sqoop_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/sqoop/", - "sqoop_tar_source": "/usr/hdp/current/sqoop-client/sqoop.tar.gz", - "tez_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/tez/", - "tez_tar_source": "/usr/hdp/current/tez-client/lib/tez.tar.gz" - }, "core-site": { "hadoop.http.authentication.simple.anonymous.allowed": "true" }, http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.0_to_2.2.4.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.0_to_2.2.4.json b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.0_to_2.2.4.json index 7721c67..81a7aa6 100755 --- a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.0_to_2.2.4.json +++ b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.0_to_2.2.4.json @@ -78,18 +78,6 @@ "yarn.scheduler.capacity.root.default-node-label-expression": " " }, "cluster-env": { - "hadoop-streaming_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/", - "hadoop-streaming_tar_source": "/usr/hdp/current/hadoop-mapreduce-client/hadoop-streaming.jar", - "hive_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/hive/", - "hive_tar_source": "/usr/hdp/current/hive-client/hive.tar.gz", - "mapreduce_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/", - "mapreduce_tar_source": "/usr/hdp/current/hadoop-client/mapreduce.tar.gz", - "pig_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/pig/", - "pig_tar_source": "/usr/hdp/current/pig-client/pig.tar.gz", - "sqoop_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/sqoop/", - "sqoop_tar_source": "/usr/hdp/current/sqoop-client/sqoop.tar.gz", - "tez_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/tez/", - "tez_tar_source": "/usr/hdp/current/tez-client/lib/tez.tar.gz", "smokeuser_principal_name": { "remove": "yes" } http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.2.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.2.json b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.2.json index a040268..183a82c 100644 --- a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.2.json +++ b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.2.json @@ -77,20 +77,6 @@ "yarn.scheduler.capacity.root.accessible-node-labels.default.maximum-capacity": "-1", "yarn.scheduler.capacity.root.default-node-label-expression": " " }, - "cluster-env": { - "hadoop-streaming_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/", - "hadoop-streaming_tar_source": "/usr/hdp/current/hadoop-mapreduce-client/hadoop-streaming.jar", - "hive_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/hive/", - "hive_tar_source": "/usr/hdp/current/hive-client/hive.tar.gz", - "mapreduce_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/", - "mapreduce_tar_source": "/usr/hdp/current/hadoop-client/mapreduce.tar.gz", - "pig_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/pig/", - "pig_tar_source": "/usr/hdp/current/pig-client/pig.tar.gz", - "sqoop_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/sqoop/", - "sqoop_tar_source": "/usr/hdp/current/sqoop-client/sqoop.tar.gz", - "tez_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/tez/", - "tez_tar_source": "/usr/hdp/current/tez-client/lib/tez.tar.gz" - }, "core-site": { "hadoop.http.authentication.simple.anonymous.allowed": "true" }, http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.4.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.4.json b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.4.json index ba2b3d7..452f6fc 100644 --- a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.4.json +++ b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.4.json @@ -84,18 +84,6 @@ "yarn.scheduler.capacity.root.default-node-label-expression": " " }, "cluster-env": { - "hadoop-streaming_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/", - "hadoop-streaming_tar_source": "/usr/hdp/current/hadoop-mapreduce-client/hadoop-streaming.jar", - "hive_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/hive/", - "hive_tar_source": "/usr/hdp/current/hive-client/hive.tar.gz", - "mapreduce_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/", - "mapreduce_tar_source": "/usr/hdp/current/hadoop-client/mapreduce.tar.gz", - "pig_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/pig/", - "pig_tar_source": "/usr/hdp/current/pig-client/pig.tar.gz", - "sqoop_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/sqoop/", - "sqoop_tar_source": "/usr/hdp/current/sqoop-client/sqoop.tar.gz", - "tez_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/tez/", - "tez_tar_source": "/usr/hdp/current/tez-client/lib/tez.tar.gz", "smokeuser_principal_name": {"remove": "yes"} }, "core-site": { http://git-wip-us.apache.org/repos/asf/ambari/blob/f1787236/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java index 804aa60..0ca4c8e 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java @@ -451,20 +451,8 @@ public class UpgradeCatalog200Test { propertiesExpectedT0.put("user_group", "hadoop"); propertiesExpectedT0.put("kinit_path_local", "/usr/bin"); propertiesExpectedT0.put("security_enabled", "true"); - propertiesExpectedT0.put("hive_tar_destination_folder", "hdfs,///hdp/apps/{{ hdp_stack_version }}/hive/"); - propertiesExpectedT0.put("sqoop_tar_source", "/usr/hdp/current/sqoop-client/sqoop.tar.gz"); - propertiesExpectedT0.put("hadoop-streaming_tar_destination_folder", "hdfs,///hdp/apps/{{ hdp_stack_version }}/mapreduce/"); - propertiesExpectedT0.put("pig_tar_source", "/usr/hdp/current/pig-client/pig.tar.gz"); - propertiesExpectedT0.put("mapreduce_tar_destination_folder", "hdfs,///hdp/apps/{{ hdp_stack_version }}/mapreduce/"); - propertiesExpectedT0.put("hive_tar_source", "/usr/hdp/current/hive-client/hive.tar.gz"); - propertiesExpectedT0.put("mapreduce_tar_source", "/usr/hdp/current/hadoop-client/mapreduce.tar.gz"); propertiesExpectedT0.put("smokeuser", "ambari-qa"); - propertiesExpectedT0.put("pig_tar_destination_folder", "hdfs,///hdp/apps/{{ hdp_stack_version }}/pig/"); - propertiesExpectedT0.put("hadoop-streaming_tar_source", "/usr/hdp/current/hadoop-mapreduce-client/hadoop-streaming.jar"); - propertiesExpectedT0.put("tez_tar_destination_folder", "hdfs,///hdp/apps/{{ hdp_stack_version }}/tez/"); propertiesExpectedT0.put("smokeuser_keytab", "/etc/security/keytabs/smokeuser.headless.keytab"); - propertiesExpectedT0.put("sqoop_tar_destination_folder", "hdfs,///hdp/apps/{{ hdp_stack_version }}/sqoop/"); - propertiesExpectedT0.put("tez_tar_source", "/usr/hdp/current/tez-client/lib/tez.tar.gz"); propertiesExpectedT0.put("ignore_groupsusers_create", "false"); final Map<String, String> propertiesExpectedT1 = new HashMap<String, String>(propertiesExpectedT0);
