Repository: ambari Updated Branches: refs/heads/trunk 3f080663a -> bffcbd285
AMBARI-16717 - Knox Gateway Uses Wrong Keystore After Upgrade (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/bffcbd28 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/bffcbd28 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/bffcbd28 Branch: refs/heads/trunk Commit: bffcbd28561ce2940ab659cbbd0fd32be81dd3f5 Parents: 3f08066 Author: Jonathan Hurley <[email protected]> Authored: Tue May 17 18:34:07 2016 -0400 Committer: Jonathan Hurley <[email protected]> Committed: Wed May 18 14:20:07 2016 -0400 ---------------------------------------------------------------------- .../package/scripts/falcon_server_upgrade.py | 20 ++--- .../1.4.0.2.0/package/scripts/flume_handler.py | 13 +-- .../1.4.0.2.0/package/scripts/flume_upgrade.py | 88 -------------------- .../0.5.0.2.2/package/scripts/knox_gateway.py | 62 ++++++-------- .../KNOX/0.5.0.2.2/package/scripts/upgrade.py | 57 ++++++++----- .../4.0.0.2.0/package/scripts/oozie_server.py | 3 - .../package/scripts/oozie_server_upgrade.py | 70 ---------------- .../HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml | 6 -- .../HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml | 6 -- .../HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml | 6 -- .../HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml | 6 -- .../HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml | 6 -- .../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml | 6 -- .../stacks/HDP/2.3/upgrades/upgrade-2.3.xml | 6 -- .../HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml | 6 -- .../HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml | 6 -- .../stacks/HDP/2.4/upgrades/upgrade-2.4.xml | 6 -- .../HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml | 6 -- .../stacks/HDP/2.5/upgrades/upgrade-2.5.xml | 6 -- .../stacks/2.0.6/OOZIE/test_oozie_server.py | 79 ++---------------- .../stacks/2.1/FALCON/test_falcon_server.py | 31 +------ .../python/stacks/2.2/KNOX/test_knox_gateway.py | 65 ++------------- 22 files changed, 86 insertions(+), 474 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/bffcbd28/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server_upgrade.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server_upgrade.py index 8eca96c..c86bedc 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server_upgrade.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server_upgrade.py @@ -17,14 +17,12 @@ limitations under the License. """ import os -import shutil -import tarfile import tempfile from resource_management.core.logger import Logger from resource_management.core.exceptions import Fail from resource_management.libraries.functions import tar_archive -from resource_management.core.resources.system import Execute, Directory +from resource_management.core.resources.system import Directory BACKUP_TEMP_DIR = "falcon-upgrade-backup" BACKUP_DATA_ARCHIVE = "falcon-local-backup.tar" @@ -32,11 +30,10 @@ BACKUP_CONF_ARCHIVE = "falcon-conf-backup.tar" def post_stop_backup(): """ - Backs up the falcon configuration and data directories as part of the - upgrade process. + Backs up falcon directories as part of the upgrade process. :return: """ - Logger.info('Backing up Falcon data and configuration directories before upgrade...') + Logger.info('Backing up Falcon directories before upgrade...') directoryMappings = _get_directory_mappings() absolute_backup_dir = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR) @@ -59,11 +56,11 @@ def post_stop_backup(): def pre_start_restore(): """ - Restores the data and configuration backups to their proper locations + Restores the directory backups to their proper locations after an upgrade has completed. :return: """ - Logger.info('Restoring Falcon data and configuration directories after upgrade...') + Logger.info('Restoring Falcon backed up directories after upgrade...') directoryMappings = _get_directory_mappings() for directory in directoryMappings: @@ -76,9 +73,7 @@ def pre_start_restore(): tar_archive.untar_archive(archive, directory) # cleanup - Directory(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR), - action = "delete", - ) + Directory(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR), action = "delete" ) def _get_directory_mappings(): @@ -89,5 +84,4 @@ def _get_directory_mappings(): """ import params - return { params.falcon_local_dir : BACKUP_DATA_ARCHIVE, - params.falcon_conf_dir : BACKUP_CONF_ARCHIVE } + return { params.falcon_local_dir : BACKUP_DATA_ARCHIVE } http://git-wip-us.apache.org/repos/asf/ambari/blob/bffcbd28/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_handler.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_handler.py b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_handler.py index 4d72463..46dce56 100644 --- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_handler.py +++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_handler.py @@ -17,22 +17,19 @@ limitations under the License. """ -import flume_upgrade - from flume import flume from flume import get_desired_state from resource_management.libraries.script.script import Script from resource_management.libraries.functions import conf_select, stack_select from resource_management.libraries.functions.flume_agent_helper import find_expected_agent_names, get_flume_status -from resource_management.libraries.functions.default import default from resource_management.core.logger import Logger from resource_management.core.resources.service import Service import service_mapping from ambari_commons import OSConst from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from resource_management.libraries.functions.stack_features import check_stack_feature -from resource_management.libraries.functions.constants import StackFeature, Direction +from resource_management.libraries.functions.constants import StackFeature class FlumeHandler(Script): def configure(self, env): @@ -61,10 +58,6 @@ class FlumeHandlerLinux(FlumeHandler): env.set_params(params) flume(action='stop') - # only backup data on upgrade - if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE: - flume_upgrade.post_stop_backup() - def status(self, env): import params env.set_params(params) @@ -98,10 +91,6 @@ class FlumeHandlerLinux(FlumeHandler): conf_select.select(params.stack_name, "flume", params.version) stack_select.select("flume-server", params.version) - # only restore on upgrade, not downgrade - if params.upgrade_direction == Direction.UPGRADE: - flume_upgrade.pre_start_restore() - def get_log_folder(self): import params return params.flume_log_dir http://git-wip-us.apache.org/repos/asf/ambari/blob/bffcbd28/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py deleted file mode 100644 index 64c0032..0000000 --- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py +++ /dev/null @@ -1,88 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import os -import shutil -import tarfile -import tempfile - -from resource_management.core.logger import Logger -from resource_management.core.exceptions import Fail -from resource_management.core.resources.system import Execute -from resource_management.libraries.functions import tar_archive - -BACKUP_TEMP_DIR = "flume-upgrade-backup" -BACKUP_CONF_DIR_ARCHIVE = "flume-conf-backup.tar" - -def post_stop_backup(): - """ - Backs up the flume config, config dir, file/spillable channels as part of the - upgrade process. - :return: - """ - Logger.info('Backing up Flume data and configuration before upgrade...') - directoryMappings = _get_directory_mappings() - - absolute_backup_dir = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR) - if not os.path.isdir(absolute_backup_dir): - os.makedirs(absolute_backup_dir) - - for directory in directoryMappings: - if not os.path.isdir(directory): - raise Fail("Unable to backup missing directory {0}".format(directory)) - - archive = os.path.join(absolute_backup_dir, directoryMappings[directory]) - Logger.info('Compressing {0} to {1}'.format(directory, archive)) - - if os.path.exists(archive): - os.remove(archive) - - # backup the directory, following symlinks instead of including them - tar_archive.archive_directory_dereference(archive, directory) - - -def pre_start_restore(): - """ - Restores the flume config, config dir, file/spillable channels to their proper locations - after an upgrade has completed. - :return: - """ - Logger.info('Restoring Flume data and configuration after upgrade...') - directoryMappings = _get_directory_mappings() - - for directory in directoryMappings: - archive = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR, - directoryMappings[directory]) - - if os.path.isfile(archive): - tar_archive.untar_archive(archive, directory) - - # cleanup - if os.path.exists(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR)): - shutil.rmtree(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR)) - - -def _get_directory_mappings(): - """ - Gets a dictionary of directory to archive name that represents the - directories that need to be backed up and their output tarball archive targets - :return: the dictionary of directory to tarball mappings - """ - import params - - return { params.flume_conf_dir : BACKUP_CONF_DIR_ARCHIVE} http://git-wip-us.apache.org/repos/asf/ambari/blob/bffcbd28/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py index e5e4103..1917369 100644 --- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py +++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py @@ -18,26 +18,26 @@ limitations under the License. """ import os -import tarfile from resource_management.libraries.script.script import Script -from resource_management.libraries.functions import conf_select, tar_archive -from resource_management.libraries.functions import stack_select from resource_management.libraries.functions.check_process_status import check_process_status from resource_management.libraries.functions import format from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select from resource_management.libraries.functions import Direction -from resource_management.libraries.functions.security_commons import build_expectations, \ - cached_kinit_executor, validate_security_config_properties, get_params_from_filesystem, \ - FILE_TYPE_XML -from resource_management.core.resources.system import File, Execute, Directory, Link +from resource_management.libraries.functions.security_commons import build_expectations +from resource_management.libraries.functions.security_commons import cached_kinit_executor +from resource_management.libraries.functions.security_commons import validate_security_config_properties +from resource_management.libraries.functions.security_commons import get_params_from_filesystem +from resource_management.libraries.functions.security_commons import FILE_TYPE_XML + +from resource_management.core.resources.system import File, Execute, Link from resource_management.core.resources.service import Service from resource_management.core.logger import Logger from resource_management.libraries.functions.show_logs import show_logs from ambari_commons import OSConst, OSCheck -from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from ambari_commons.os_family_impl import OsFamilyImpl if OSCheck.is_windows_family(): from resource_management.libraries.functions.windows_service_utils import check_windows_service_status @@ -46,8 +46,6 @@ import upgrade from knox import knox, update_knox_logfolder_permissions from knox_ldap import ldap from setup_ranger_knox import setup_ranger_knox -from resource_management.libraries.functions.stack_features import check_stack_feature -from resource_management.libraries.functions import StackFeature class KnoxGateway(Script): @@ -114,34 +112,22 @@ class KnoxGatewayDefault(KnoxGateway): def pre_upgrade_restart(self, env, upgrade_type=None): import params env.set_params(params) - if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): - absolute_backup_dir = None - if params.upgrade_direction and params.upgrade_direction == Direction.UPGRADE: - Logger.info("Backing up directories. Initial conf folder: %s" % os.path.realpath(params.knox_conf_dir)) - - # This will backup the contents of the conf directory into /tmp/knox-upgrade-backup/knox-conf-backup.tar - absolute_backup_dir = upgrade.backup_data() - - # <conf-selector-tool> will change the symlink to the conf folder. - conf_select.select(params.stack_name, "knox", params.version) - stack_select.select("knox-server", params.version) - - # Extract the tar of the old conf folder into the new conf directory - if absolute_backup_dir is not None and params.upgrade_direction and params.upgrade_direction == Direction.UPGRADE: - conf_tar_source_path = os.path.join(absolute_backup_dir, upgrade.BACKUP_CONF_ARCHIVE) - if os.path.exists(conf_tar_source_path): - extract_dir = os.path.realpath(params.knox_conf_dir) - conf_tar_dest_path = os.path.join(extract_dir, upgrade.BACKUP_CONF_ARCHIVE) - Logger.info("Copying %s into %s file." % (upgrade.BACKUP_CONF_ARCHIVE, conf_tar_dest_path)) - Execute(('cp', conf_tar_source_path, conf_tar_dest_path), - sudo = True, - ) - - tar_archive.untar_archive(conf_tar_source_path, extract_dir) - - File(conf_tar_dest_path, - action = "delete", - ) + + # backup the data directory to /tmp/knox-upgrade-backup/knox-data-backup.tar just in case + # something happens; Knox is interesting in that they re-generate missing files like + # keystores which can cause side effects if the upgrade goes wrong + if params.upgrade_direction and params.upgrade_direction == Direction.UPGRADE: + absolute_backup_dir = upgrade.backup_data() + Logger.info("Knox data was successfully backed up to {0}".format(absolute_backup_dir)) + + # <conf-selector-tool> will change the symlink to the conf folder. + conf_select.select(params.stack_name, "knox", params.version) + stack_select.select("knox-server", params.version) + + # seed the new Knox data directory with the keystores of yesteryear + if params.upgrade_direction == Direction.UPGRADE: + upgrade.seed_current_data_directory() + def start(self, env, upgrade_type=None): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/bffcbd28/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py index 63949f8..8c71260 100644 --- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py +++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py @@ -19,21 +19,19 @@ limitations under the License. """ import os -import tarfile import tempfile from resource_management.core.logger import Logger from resource_management.core.exceptions import Fail +from resource_management.core.resources.system import Execute from resource_management.libraries.functions import tar_archive from resource_management.libraries.functions import format -from resource_management.libraries.functions import Direction from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions import StackFeature BACKUP_TEMP_DIR = "knox-upgrade-backup" BACKUP_DATA_ARCHIVE = "knox-data-backup.tar" -BACKUP_CONF_ARCHIVE = "knox-conf-backup.tar" def backup_data(): """ @@ -65,6 +63,39 @@ def backup_data(): return absolute_backup_dir +def seed_current_data_directory(): + """ + Knox uses "versioned" data directories in some stacks: + /usr/hdp/2.2.0.0-1234/knox/data -> /var/lib/knox/data + /usr/hdp/2.3.0.0-4567/knox/data -> /var/lib/knox/data-2.3.0.0-4567 + + If the stack being upgraded to supports versioned data directories for Knox, then we should + seed the data from the prior version. This is mainly because Knox keeps things like keystores + in the data directory and if those aren't copied over then it will re-create self-signed + versions. This side-effect behavior causes loss of service in clusters where Knox is using + custom keystores. + + cp -R -p -f /usr/hdp/<old>/knox-server/data/. /usr/hdp/current/knox-server/data + :return: + """ + import params + + if params.version is None or params.upgrade_from_version is None: + raise Fail("The source and target versions are required") + + if check_stack_feature(StackFeature.KNOX_VERSIONED_DATA_DIR, params.version): + Logger.info("Seeding Knox data from prior version...") + + # <stack-root>/2.3.0.0-1234/knox/data/. + source_data_dir = os.path.join(params.stack_root, params.upgrade_from_version, "knox", "data", ".") + + # <stack-root>/current/knox-server/data + target_data_dir = os.path.join(params.stack_root, "current", "knox-server", "data") + + # recursive copy, overwriting, and preserving attributes + Execute(("cp", "-R", "-p", "-f", source_data_dir, target_data_dir), sudo = True) + + def _get_directory_mappings_during_upgrade(): """ Gets a dictionary of directory to archive name that represents the @@ -73,23 +104,11 @@ def _get_directory_mappings_during_upgrade(): """ import params - # Must be performing an Upgrade - if params.upgrade_direction is None or params.upgrade_direction != Direction.UPGRADE or \ - params.upgrade_from_version is None or params.upgrade_from_version == "": - Logger.error("Function _get_directory_mappings_during_upgrade() can only be called during a Stack Upgrade in direction UPGRADE.") - return {} - - # By default, use this for all stacks. - knox_data_dir = '/var/lib/knox/data' + # the data directory is always a symlink to the "correct" data directory in /var/lib/knox + # such as /var/lib/knox/data or /var/lib/knox/data-2.4.0.0-1234 + knox_data_dir = '/usr/hdp/current/knox-server/data' - if params.upgrade_from_version and check_stack_feature(StackFeature.KNOX_VERSIONED_DATA_DIR, params.upgrade_from_version): - # Use the version that is being upgraded from. - knox_data_dir = format('{stack_root}/{upgrade_from_version}/knox/data') - - - directories = {knox_data_dir: BACKUP_DATA_ARCHIVE, - params.knox_conf_dir + "/": BACKUP_CONF_ARCHIVE} # the trailing "/" is important here so as to not include the "conf" folder itself + directories = { knox_data_dir: BACKUP_DATA_ARCHIVE } Logger.info(format("Knox directories to backup:\n{directories}")) return directories - http://git-wip-us.apache.org/repos/asf/ambari/blob/bffcbd28/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py index fcce418..66bcdd7 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py @@ -192,13 +192,10 @@ class OozieServerDefault(OozieServer): Logger.info("Executing Oozie Server Stack Upgrade pre-restart") - OozieUpgrade.backup_configuration() - if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): conf_select.select(params.stack_name, "oozie", params.version) stack_select.select("oozie-server", params.version) - OozieUpgrade.restore_configuration() OozieUpgrade.prepare_libext_directory() def get_log_folder(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/bffcbd28/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py index 28d2991..a1f3336 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py @@ -19,9 +19,7 @@ limitations under the License. import glob import os import shutil -import tempfile -from resource_management.core import shell from resource_management.core.logger import Logger from resource_management.core.exceptions import Fail from resource_management.core.resources.system import Execute @@ -30,7 +28,6 @@ from resource_management.core.resources.system import File from resource_management.libraries.functions import Direction from resource_management.libraries.functions import format from resource_management.libraries.functions import stack_select -from resource_management.libraries.functions import tar_archive from resource_management.libraries.functions.oozie_prepare_war import prepare_war from resource_management.libraries.script.script import Script from resource_management.libraries.functions import StackFeature @@ -44,59 +41,6 @@ BACKUP_CONF_ARCHIVE = "oozie-conf-backup.tar" class OozieUpgrade(Script): @staticmethod - def backup_configuration(): - """ - Backs up the oozie configuration as part of the upgrade process. - :return: - """ - Logger.info('Backing up Oozie configuration directory before upgrade...') - directoryMappings = OozieUpgrade._get_directory_mappings() - - absolute_backup_dir = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR) - if not os.path.isdir(absolute_backup_dir): - os.makedirs(absolute_backup_dir) - - for directory in directoryMappings: - if not os.path.isdir(directory): - raise Fail("Unable to backup missing directory {0}".format(directory)) - - archive = os.path.join(absolute_backup_dir, directoryMappings[directory]) - Logger.info('Compressing {0} to {1}'.format(directory, archive)) - - if os.path.exists(archive): - os.remove(archive) - - # backup the directory, following symlinks instead of including them - tar_archive.archive_directory_dereference(archive, directory) - - - @staticmethod - def restore_configuration(): - """ - Restores the configuration backups to their proper locations after an - upgrade has completed. - :return: - """ - Logger.info('Restoring Oozie configuration directory after upgrade...') - directoryMappings = OozieUpgrade._get_directory_mappings() - - for directory in directoryMappings: - archive = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR, - directoryMappings[directory]) - - if not os.path.isfile(archive): - raise Fail("Unable to restore missing backup archive {0}".format(archive)) - - Logger.info('Extracting {0} to {1}'.format(archive, directory)) - - tar_archive.untar_archive(archive, directory) - - # cleanup - Directory(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR), - action="delete", - ) - - @staticmethod def prepare_libext_directory(): """ Performs the following actions on libext: @@ -289,19 +233,5 @@ class OozieUpgrade(Script): Execute(sharelib_command, user=params.oozie_user, logoutput=True) - - @staticmethod - def _get_directory_mappings(): - """ - Gets a dictionary of directory to archive name that represents the - directories that need to be backed up and their output tarball archive targets - :return: the dictionary of directory to tarball mappings - """ - import params - - # the trailing "/" is important here so as to not include the "conf" folder itself - return { params.conf_dir + "/" : BACKUP_CONF_ARCHIVE } - - if __name__ == "__main__": OozieUpgrade().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/bffcbd28/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml index 5f0c6aa..d4cb742 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml @@ -134,12 +134,6 @@ </task> </execute-stage> - <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Backup Knox Data"> - <task xsi:type="manual"> - <message>Before continuing, please backup the Knox data. E.g., "cp -RL /var/lib/knox/data/* ~/knox_backup/" on the following host(s): {{hosts.all}}.</message> - </task> - </execute-stage> - <execute-stage service="HBASE" component="HBASE_MASTER" title="Snapshot HBASE"> <task xsi:type="execute" hosts="master"> <script>scripts/hbase_upgrade.py</script> http://git-wip-us.apache.org/repos/asf/ambari/blob/bffcbd28/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml index cf79368..4fd5801 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml @@ -145,12 +145,6 @@ </task> </execute-stage> - <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Backup Knox Data"> - <task xsi:type="manual"> - <message>Before continuing, please backup the Knox data. E.g., "cp -RL /var/lib/knox/data/* ~/knox_backup/" on the following host(s): {{hosts.all}}.</message> - </task> - </execute-stage> - <execute-stage service="HBASE" component="HBASE_MASTER" title="Snapshot HBASE"> <task xsi:type="execute" hosts="master"> <script>scripts/hbase_upgrade.py</script> http://git-wip-us.apache.org/repos/asf/ambari/blob/bffcbd28/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml index fab12c8..272a3cc 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml @@ -157,12 +157,6 @@ </task> </execute-stage> - <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Backup Knox Data"> - <task xsi:type="manual"> - <message>Before continuing, please backup the Knox data. E.g., "cp -RL /usr/hdp/current/knox-server/data/* ~/knox_backup/" on the following host(s): {{hosts.all}}.</message> - </task> - </execute-stage> - <execute-stage service="HBASE" component="HBASE_MASTER" title="Snapshot HBASE"> <task xsi:type="execute" hosts="master"> <script>scripts/hbase_upgrade.py</script> http://git-wip-us.apache.org/repos/asf/ambari/blob/bffcbd28/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml index 1887414..111b432 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml @@ -147,12 +147,6 @@ </task> </execute-stage> - <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Backup Knox Data"> - <task xsi:type="manual"> - <message>Before continuing, please backup the Knox data. E.g., "cp -RL /usr/hdp/current/knox-server/data/* ~/knox_backup/" on the following host(s): {{hosts.all}}.</message> - </task> - </execute-stage> - <execute-stage service="HBASE" component="HBASE_MASTER" title="Snapshot HBASE"> <task xsi:type="execute" hosts="master"> <script>scripts/hbase_upgrade.py</script> http://git-wip-us.apache.org/repos/asf/ambari/blob/bffcbd28/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml index 469f2e7..9365646 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml @@ -147,12 +147,6 @@ </task> </execute-stage> - <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Backup Knox Data"> - <task xsi:type="manual"> - <message>Before continuing, please backup the Knox data. E.g., "cp -RL /usr/hdp/current/knox-server/data/* ~/knox_backup/" on the following host(s): {{hosts.all}}.</message> - </task> - </execute-stage> - <execute-stage service="HBASE" component="HBASE_MASTER" title="Snapshot HBASE"> <task xsi:type="execute" hosts="master"> <script>scripts/hbase_upgrade.py</script> http://git-wip-us.apache.org/repos/asf/ambari/blob/bffcbd28/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml index 56cd6d0..dd033b8 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml @@ -155,12 +155,6 @@ </task> </execute-stage> - <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Backup Knox Data"> - <task xsi:type="manual"> - <message>Before continuing, please backup the Knox data. E.g., "cp -RL /usr/hdp/current/knox-server/data/* ~/knox_backup/" on the following host(s): {{hosts.all}}.</message> - </task> - </execute-stage> - <execute-stage service="HBASE" component="HBASE_MASTER" title="Snapshot HBASE"> <task xsi:type="execute" hosts="master"> <script>scripts/hbase_upgrade.py</script> http://git-wip-us.apache.org/repos/asf/ambari/blob/bffcbd28/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml index 6b74af0..614487e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml @@ -76,12 +76,6 @@ <skippable>true</skippable> <supports-auto-skip-failure>false</supports-auto-skip-failure> - <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Backup Knox Data"> - <task xsi:type="manual"> - <message>Before continuing, please backup the Knox data. E.g., "cp -RL /usr/hdp/current/knox-server/data/* ~/knox_backup/" on the following host(s): {{hosts.all}}.</message> - </task> - </execute-stage> - <execute-stage service="HBASE" component="HBASE_MASTER" title="Pre Upgrade HBase Backup"> <task xsi:type="execute" hosts="master"> <script>scripts/hbase_upgrade.py</script> http://git-wip-us.apache.org/repos/asf/ambari/blob/bffcbd28/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml index 29ebeff..e83b54b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml @@ -148,12 +148,6 @@ </task> </execute-stage> - <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Backup Knox Data"> - <task xsi:type="manual"> - <message>Before continuing, please backup the Knox data. E.g., "cp -RL /usr/hdp/current/knox-server/data/* ~/knox_backup/" on the following host(s): {{hosts.all}}.</message> - </task> - </execute-stage> - <execute-stage service="HBASE" component="HBASE_MASTER" title="Snapshot HBASE"> <task xsi:type="execute" hosts="master"> <script>scripts/hbase_upgrade.py</script> http://git-wip-us.apache.org/repos/asf/ambari/blob/bffcbd28/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml index 7d67f8e..625b851 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml @@ -150,12 +150,6 @@ </task> </execute-stage> - <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Backup Knox Data"> - <task xsi:type="manual"> - <message>Before continuing, please backup the Knox data. E.g., "cp -RL /usr/hdp/current/knox-server/data/* ~/knox_backup/" on the following host(s): {{hosts.all}}.</message> - </task> - </execute-stage> - <execute-stage service="HBASE" component="HBASE_MASTER" title="Snapshot HBASE"> <task xsi:type="execute" hosts="master"> <script>scripts/hbase_upgrade.py</script> http://git-wip-us.apache.org/repos/asf/ambari/blob/bffcbd28/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml index e3bc7a3..a9ed58d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml @@ -74,12 +74,6 @@ <skippable>true</skippable> <supports-auto-skip-failure>false</supports-auto-skip-failure> - <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Backup Knox Data"> - <task xsi:type="manual"> - <message>Before continuing, please backup the Knox data. E.g., "cp -RL /usr/hdp/current/knox-server/data/* ~/knox_backup/" on the following host(s): {{hosts.all}}.</message> - </task> - </execute-stage> - <execute-stage service="HBASE" component="HBASE_MASTER" title="Pre Upgrade HBase Backup"> <task xsi:type="execute" hosts="master"> <script>scripts/hbase_upgrade.py</script> http://git-wip-us.apache.org/repos/asf/ambari/blob/bffcbd28/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml index 5616cb4..460e6b3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml @@ -149,12 +149,6 @@ </task> </execute-stage> - <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Backup Knox Data"> - <task xsi:type="manual"> - <message>Before continuing, please backup the Knox data. E.g., "cp -RL /usr/hdp/current/knox-server/data/* ~/knox_backup/" on the following host(s): {{hosts.all}}.</message> - </task> - </execute-stage> - <execute-stage service="HBASE" component="HBASE_MASTER" title="Snapshot HBASE"> <task xsi:type="execute" hosts="master"> <script>scripts/hbase_upgrade.py</script> http://git-wip-us.apache.org/repos/asf/ambari/blob/bffcbd28/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml index d755516..23b4ca7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml @@ -74,12 +74,6 @@ <skippable>true</skippable> <supports-auto-skip-failure>false</supports-auto-skip-failure> - <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Backup Knox Data"> - <task xsi:type="manual"> - <message>Before continuing, please backup the Knox data. E.g., "cp -RL /usr/hdp/current/knox-server/data/* ~/knox_backup/" on the following host(s): {{hosts.all}}.</message> - </task> - </execute-stage> - <execute-stage service="HBASE" component="HBASE_MASTER" title="Pre Upgrade HBase Backup"> <task xsi:type="execute" hosts="master"> <script>scripts/hbase_upgrade.py</script> http://git-wip-us.apache.org/repos/asf/ambari/blob/bffcbd28/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py index ef97d20..e43f6dc 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py @@ -1234,35 +1234,20 @@ class TestOozieServer(RMFTestCase): call_mocks = [(0, prepare_war_stdout)]) self.assertTrue(isfile_mock.called) - self.assertEqual(isfile_mock.call_count,3) + self.assertEqual(isfile_mock.call_count,2) isfile_mock.assert_called_with('/usr/share/HDP-oozie/ext-2.2.zip') self.assertTrue(glob_mock.called) self.assertEqual(glob_mock.call_count,1) glob_mock.assert_called_with('/usr/hdp/2.2.1.0-2135/hadoop/lib/hadoop-lzo*.jar') - self.assertResourceCalled('Execute', - ('tar', '-zcvhf', '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar', '/usr/hdp/current/oozie-server/conf/'), - sudo = True, - tries = 3, - try_sleep = 1 - ) self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'oozie-server', u'2.2.1.0-2135'), sudo = True ) - self.assertResourceCalled('Execute', - ('tar', '-xvf', '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar', '-C', '/usr/hdp/current/oozie-server/conf//'), - sudo = True, - tries = 3, - try_sleep = 1 - ) - self.assertResourceCalled('Directory', '/tmp/oozie-upgrade-backup', action = ['delete']) self.assertResourceCalled('Directory', '/usr/hdp/current/oozie-server/libext', mode = 0777) self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/hdp/current/oozie-server/libext'), sudo=True) self.assertResourceCalled('Execute', ('chown', 'oozie:hadoop', '/usr/hdp/current/oozie-server/libext/ext-2.2.zip'), sudo=True) - self.assertResourceCalled('File', '/usr/hdp/current/oozie-server/libext/ext-2.2.zip', - mode = 0644, - ) + self.assertResourceCalled('File', '/usr/hdp/current/oozie-server/libext/ext-2.2.zip', mode = 0644) self.assertNoMoreResources() @patch("os.path.isdir") @@ -1306,39 +1291,23 @@ class TestOozieServer(RMFTestCase): mocks_dict = mocks_dict) self.assertTrue(isfile_mock.called) - self.assertEqual(isfile_mock.call_count,3) + self.assertEqual(isfile_mock.call_count,2) isfile_mock.assert_called_with('/usr/share/HDP-oozie/ext-2.2.zip') self.assertTrue(glob_mock.called) self.assertEqual(glob_mock.call_count,1) glob_mock.assert_called_with('/usr/hdp/2.3.0.0-1234/hadoop/lib/hadoop-lzo*.jar') - self.assertResourceCalled('Execute', - ('tar', '-zcvhf', '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar', '/usr/hdp/current/oozie-server/conf/'), - sudo = True, - tries = 3, - try_sleep = 1 - ) self.assertResourceCalled('Link', '/etc/oozie/conf', to = '/usr/hdp/current/oozie-client/conf', ) self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'oozie-server', '2.3.0.0-1234'), sudo = True) - self.assertResourceCalled('Execute', - ('tar', '-xvf', '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar', '-C', '/usr/hdp/current/oozie-server/conf//'), - sudo = True, - tries = 3, - try_sleep = 1 - ) - - self.assertResourceCalled('Directory', '/tmp/oozie-upgrade-backup', action = ['delete']) self.assertResourceCalled('Directory', '/usr/hdp/current/oozie-server/libext', mode = 0777) self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/hdp/current/oozie-server/libext'), sudo=True) self.assertResourceCalled('Execute', ('chown', 'oozie:hadoop', '/usr/hdp/current/oozie-server/libext/ext-2.2.zip'), sudo=True) - self.assertResourceCalled('File', '/usr/hdp/current/oozie-server/libext/ext-2.2.zip', - mode = 0644, - ) + self.assertResourceCalled('File', '/usr/hdp/current/oozie-server/libext/ext-2.2.zip', mode = 0644) self.assertNoMoreResources() self.assertEquals(1, mocks_dict['call'].call_count) @@ -1376,31 +1345,15 @@ class TestOozieServer(RMFTestCase): call_mocks = [(0, prepare_war_stdout)]) self.assertTrue(isfile_mock.called) - self.assertEqual(isfile_mock.call_count,2) + self.assertEqual(isfile_mock.call_count,1) isfile_mock.assert_called_with('/usr/share/HDP-oozie/ext-2.2.zip') - self.assertResourceCalled('Execute', - ('tar', '-zcvhf', '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar', '/usr/hdp/current/oozie-server/conf/'), - sudo = True, - tries = 3, - try_sleep = 1 - ) self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'oozie-server', u'2.2.0.0-0000'), sudo = True) - self.assertResourceCalled('Execute', - ('tar', '-xvf', '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar', '-C', '/usr/hdp/current/oozie-server/conf//'), - sudo = True, - tries = 3, - try_sleep = 1 - ) - - self.assertResourceCalled('Directory', '/tmp/oozie-upgrade-backup', action = ['delete']) self.assertResourceCalled('Directory', '/usr/hdp/current/oozie-server/libext',mode = 0777) self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/hdp/current/oozie-server/libext'), sudo=True) self.assertResourceCalled('Execute', ('chown', 'oozie:hadoop', '/usr/hdp/current/oozie-server/libext/ext-2.2.zip'), sudo=True) - self.assertResourceCalled('File', '/usr/hdp/current/oozie-server/libext/ext-2.2.zip', - mode = 0644, - ) + self.assertResourceCalled('File', '/usr/hdp/current/oozie-server/libext/ext-2.2.zip',mode = 0644) self.assertNoMoreResources() @@ -1581,32 +1534,16 @@ class TestOozieServer(RMFTestCase): mocks_dict = mocks_dict) self.assertTrue(isfile_mock.called) - self.assertEqual(isfile_mock.call_count,3) + self.assertEqual(isfile_mock.call_count,2) isfile_mock.assert_called_with('/usr/share/HDP-oozie/ext-2.2.zip') self.assertTrue(glob_mock.called) self.assertEqual(glob_mock.call_count,1) glob_mock.assert_called_with('/usr/hdp/2.3.0.0-1234/hadoop/lib/hadoop-lzo*.jar') - self.assertResourceCalled('Execute', - ('tar', '-zcvhf', '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar', '/usr/hdp/current/oozie-server/conf/'), - sudo = True, - tries = 3, - try_sleep = 1 - ) - self.assertResourceCalled('Link', '/etc/oozie/conf', - to = '/usr/hdp/current/oozie-client/conf', - ) + self.assertResourceCalled('Link', '/etc/oozie/conf', to = '/usr/hdp/current/oozie-client/conf') self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'oozie-server', '2.3.0.0-1234'), sudo = True) - self.assertResourceCalled('Execute', - ('tar', '-xvf', '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar', '-C', '/usr/hdp/current/oozie-server/conf//'), - sudo = True, - tries = 3, - try_sleep = 1 - ) - - self.assertResourceCalled('Directory', '/tmp/oozie-upgrade-backup', action = ['delete']) self.assertResourceCalled('Directory', '/usr/hdp/current/oozie-server/libext', mode = 0777) self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/hdp/current/oozie-server/libext'), sudo=True) http://git-wip-us.apache.org/repos/asf/ambari/blob/bffcbd28/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py index 6dfb609..298480c 100644 --- a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py +++ b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py @@ -200,7 +200,7 @@ class TestFalconServer(RMFTestCase): def test_upgrade(self, isfile_mock, exists_mock, isdir_mock): isdir_mock.return_value = True - exists_mock.side_effect = [False,False,True, True,True] + exists_mock.side_effect = [False,True, True, True] isfile_mock.return_value = True self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py", @@ -218,12 +218,6 @@ class TestFalconServer(RMFTestCase): self.assertResourceCalled('Execute', ('tar', '-zcvhf', - '/tmp/falcon-upgrade-backup/falcon-conf-backup.tar', - '/usr/hdp/current/falcon-server/conf'), - sudo = True, tries = 3, try_sleep = 1, - ) - self.assertResourceCalled('Execute', ('tar', - '-zcvhf', '/tmp/falcon-upgrade-backup/falcon-local-backup.tar', u'/hadoop/falcon'), sudo = True, tries = 3, try_sleep = 1, @@ -233,13 +227,6 @@ class TestFalconServer(RMFTestCase): ) self.assertResourceCalled('Execute', ('tar', '-xvf', - '/tmp/falcon-upgrade-backup/falcon-conf-backup.tar', - '-C', - '/usr/hdp/current/falcon-server/conf/'), - sudo = True, tries = 3, try_sleep = 1, - ) - self.assertResourceCalled('Execute', ('tar', - '-xvf', '/tmp/falcon-upgrade-backup/falcon-local-backup.tar', '-C', u'/hadoop/falcon/'), @@ -522,15 +509,6 @@ class TestFalconServer(RMFTestCase): ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'falcon-server', version), sudo=True,) self.assertResourceCalled('Execute', ('tar', '-xvf', - '/tmp/falcon-upgrade-backup/falcon-conf-backup.tar', - '-C', - '/usr/hdp/current/falcon-server/conf/'), - tries = 3, - sudo = True, - try_sleep = 1, - ) - self.assertResourceCalled('Execute', ('tar', - '-xvf', '/tmp/falcon-upgrade-backup/falcon-local-backup.tar', '-C', u'/hadoop/falcon/'), @@ -571,13 +549,6 @@ class TestFalconServer(RMFTestCase): self.assertResourceCalled('Execute', ('tar', '-xvf', - '/tmp/falcon-upgrade-backup/falcon-conf-backup.tar', - '-C', - '/usr/hdp/current/falcon-server/conf/'), - sudo = True, tries = 3, try_sleep = 1, - ) - self.assertResourceCalled('Execute', ('tar', - '-xvf', '/tmp/falcon-upgrade-backup/falcon-local-backup.tar', '-C', u'/hadoop/falcon/'), http://git-wip-us.apache.org/repos/asf/ambari/blob/bffcbd28/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py b/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py index fbc55ca..1e4a708 100644 --- a/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py +++ b/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py @@ -248,14 +248,8 @@ class TestKnoxGateway(RMFTestCase): self.assertResourceCalled('Execute', ('tar', '-zcvhf', - '/tmp/knox-upgrade-backup/knox-conf-backup.tar', - '/usr/hdp/current/knox-server/conf/'), - sudo = True, tries = 3, try_sleep = 1, - ) - self.assertResourceCalled('Execute', ('tar', - '-zcvhf', '/tmp/knox-upgrade-backup/knox-data-backup.tar', - '/var/lib/knox/data'), + '/usr/hdp/current/knox-server/data'), sudo = True, tries = 3, try_sleep = 1, ) self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'knox-server', '2.2.1.0-3242'), @@ -292,32 +286,11 @@ class TestKnoxGateway(RMFTestCase): self.assertResourceCalled('Execute', ('tar', '-zcvhf', - '/tmp/knox-upgrade-backup/knox-conf-backup.tar', - '/usr/hdp/current/knox-server/conf/'), - sudo = True, tries = 3, try_sleep = 1, - ) - self.assertResourceCalled('Execute', ('tar', - '-zcvhf', '/tmp/knox-upgrade-backup/knox-data-backup.tar', - '/var/lib/knox/data'), + '/usr/hdp/current/knox-server/data'), sudo = True, tries = 3, try_sleep = 1, ) self.assertResourceCalledIgnoreEarlier('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'knox-server', version),sudo = True) - self.assertResourceCalled('Execute', ('cp', - '/tmp/knox-upgrade-backup/knox-conf-backup.tar', - '/usr/hdp/current/knox-server/conf/knox-conf-backup.tar'), - sudo = True, - ) - self.assertResourceCalled('Execute', ('tar', - '-xvf', - '/tmp/knox-upgrade-backup/knox-conf-backup.tar', - '-C', - '/usr/hdp/current/knox-server/conf/'), - sudo = True, tries = 3, try_sleep = 1, - ) - self.assertResourceCalled('File', '/usr/hdp/current/knox-server/conf/knox-conf-backup.tar', - action = ['delete'], - ) self.assertNoMoreResources() self.assertEquals(1, mocks_dict['call'].call_count) @@ -362,32 +335,14 @@ class TestKnoxGateway(RMFTestCase): self.assertResourceCalled('Execute', ('tar', '-zcvhf', - '/tmp/knox-upgrade-backup/knox-conf-backup.tar', - '/usr/hdp/current/knox-server/conf/'), - sudo = True, tries = 3, try_sleep = 1, - ) - self.assertResourceCalled('Execute', ('tar', - '-zcvhf', '/tmp/knox-upgrade-backup/knox-data-backup.tar', - '/var/lib/knox/data'), + '/usr/hdp/current/knox-server/data'), sudo = True, tries = 3, try_sleep = 1, ) self.assertResourceCalledIgnoreEarlier('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'knox-server', version),sudo = True) - self.assertResourceCalled('Execute', ('cp', - '/tmp/knox-upgrade-backup/knox-conf-backup.tar', - '/usr/hdp/current/knox-server/conf/knox-conf-backup.tar'), - sudo = True, - ) - self.assertResourceCalled('Execute', ('tar', - '-xvf', - '/tmp/knox-upgrade-backup/knox-conf-backup.tar', - '-C', - '/usr/hdp/current/knox-server/conf/'), - sudo = True, tries = 3, try_sleep = 1, - ) - self.assertResourceCalled('File', '/usr/hdp/current/knox-server/conf/knox-conf-backup.tar', - action = ['delete'], - ) + + self.assertResourceCalled('Execute', ("cp", "-R", "-p", "-f", "/usr/hdp/2.3.0.0-1234/knox/data/.", "/usr/hdp/current/knox-server/data"), sudo = True) + self.assertNoMoreResources() self.assertEquals(1, mocks_dict['call'].call_count) @@ -432,14 +387,8 @@ class TestKnoxGateway(RMFTestCase): self.assertResourceCalled('Execute', ('tar', '-zcvhf', - '/tmp/knox-upgrade-backup/knox-conf-backup.tar', - '/usr/hdp/current/knox-server/conf/'), - sudo = True, tries = 3, try_sleep = 1, - ) - self.assertResourceCalled('Execute', ('tar', - '-zcvhf', '/tmp/knox-upgrade-backup/knox-data-backup.tar', - "/usr/hdp/%s/knox/data" % source_version), + "/usr/hdp/current/knox-server/data"), sudo = True, tries = 3, try_sleep = 1, )
