AMBARI-9072 - Upgrade Pack for Oozie (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a8fc971b Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a8fc971b Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a8fc971b Branch: refs/heads/trunk Commit: a8fc971b1af3367c7d4054aa3c2cca9ba1c16243 Parents: 8c68301 Author: Jonathan Hurley <[email protected]> Authored: Fri Jan 9 13:05:56 2015 -0500 Committer: Jonathan Hurley <[email protected]> Committed: Fri Jan 9 15:22:38 2015 -0500 ---------------------------------------------------------------------- .../4.0.0.2.0/package/scripts/oozie_client.py | 17 +- .../4.0.0.2.0/package/scripts/oozie_server.py | 60 +- .../package/scripts/oozie_server_upgrade.py | 192 +++++ .../4.0.0.2.0/package/scripts/oozie_service.py | 68 +- .../OOZIE/4.0.0.2.0/package/scripts/params.py | 49 +- .../stacks/HDP/2.2/upgrades/upgrade-2.2.xml | 30 +- .../stacks/2.0.6/OOZIE/test_oozie_server.py | 88 +++ .../stacks/2.2/configs/falcon-upgrade.json | 759 +------------------ .../python/stacks/2.2/configs/hive-upgrade.json | 337 -------- .../stacks/2.2/configs/oozie-upgrade.json | 297 ++++++++ 10 files changed, 743 insertions(+), 1154 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a8fc971b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_client.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_client.py index a7313f3..9a3eb66 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_client.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_client.py @@ -33,7 +33,8 @@ class OozieClient(Script): def install(self, env): self.install_packages(env) self.configure(env) - + + def configure(self, env): import params env.set_params(params) @@ -44,6 +45,20 @@ class OozieClient(Script): def status(self, env): raise ClientComponentHasNoStatus() + + + def pre_rolling_restart(self, env): + import params + env.set_params(params) + + # this function should not execute if the version can't be determined or + # is not at least HDP 2.2.0.0 + if not params.version or compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') < 0: + return + + Logger.info("Executing Oozie Client Rolling Upgrade pre-restart") + Execute(format("hdp-select set oozie-client {version}")) + if __name__ == "__main__": OozieClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/a8fc971b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py index a129bce..a43bda5 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py @@ -18,11 +18,20 @@ limitations under the License. """ -import sys -from resource_management import * -from resource_management.libraries.functions.security_commons import build_expectations, \ - cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ - FILE_TYPE_XML +import oozie_server_upgrade + +from resource_management.core import Logger +from resource_management.core.resources.system import Execute +from resource_management.libraries.functions import format +from resource_management.libraries.script import Script +from resource_management.libraries.functions import check_process_status +from resource_management.libraries.functions import compare_versions +from resource_management.libraries.functions import format_hdp_stack_version +from resource_management.libraries.functions.security_commons import build_expectations +from resource_management.libraries.functions.security_commons import cached_kinit_executor +from resource_management.libraries.functions.security_commons import get_params_from_filesystem +from resource_management.libraries.functions.security_commons import validate_security_config_properties +from resource_management.libraries.functions.security_commons import FILE_TYPE_XML from oozie import oozie from oozie_service import oozie_service @@ -35,26 +44,30 @@ class OozieServer(Script): def install(self, env): self.install_packages(env) - + + def configure(self, env): import params env.set_params(params) oozie(is_server=True) - + + def start(self, env, rolling_restart=False): import params env.set_params(params) #TODO remove this when config command will be implemented self.configure(env) - oozie_service(action='start') + + oozie_service(action='start', rolling_restart=rolling_restart) self.save_component_version_to_structured_out(params.stack_name) def stop(self, env, rolling_restart=False): import params env.set_params(params) - oozie_service(action='stop') + oozie_service(action='stop', rolling_restart=rolling_restart) + def status(self, env): import status_params @@ -126,5 +139,34 @@ class OozieServer(Script): else: self.put_structured_out({"securityState": "UNSECURED"}) + + def pre_rolling_restart(self, env): + """ + Performs the tasks surrounding the Oozie startup when a rolling upgrade + is in progress. This includes backing up the configuration, updating + the database, preparing the WAR, and installing the sharelib in HDFS. + :param env: + :return: + """ + import params + env.set_params(params) + + # this function should not execute if the version can't be determined or + # is not at least HDP 2.2.0.0 + if not params.version or compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') < 0: + return + + Logger.info("Executing Oozie Server Rolling Upgrade pre-restart") + + oozie_server_upgrade.backup_configuration() + oozie_server_upgrade.pre_hdp_select() + + Execute(format("hdp-select set oozie-server {version}")) + + oozie_server_upgrade.restore_configuration() + oozie_server_upgrade.prepare_libext_directory() + oozie_server_upgrade.upgrade_oozie() + + if __name__ == "__main__": OozieServer().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/a8fc971b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py new file mode 100644 index 0000000..b2d9703 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py @@ -0,0 +1,192 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +import glob +import os +import shutil +import tarfile +import tempfile + +from resource_management.core import shell +from resource_management.core.logger import Logger +from resource_management.core.exceptions import Fail +from resource_management.core.resources.system import Execute +from resource_management.libraries.functions import format + +BACKUP_TEMP_DIR = "oozie-upgrade-backup" +BACKUP_CONF_ARCHIVE = "oozie-conf-backup.tar" + + +def backup_configuration(): + """ + Backs up the oozie configuration as part of the upgrade process. + :return: + """ + Logger.info('Backing up Oozie configuration directory before upgrade...') + directoryMappings = _get_directory_mappings() + + absolute_backup_dir = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR) + if not os.path.isdir(absolute_backup_dir): + os.makedirs(absolute_backup_dir) + + for directory in directoryMappings: + if not os.path.isdir(directory): + raise Fail("Unable to backup missing directory {0}".format(directory)) + + archive = os.path.join(absolute_backup_dir, directoryMappings[directory]) + Logger.info('Compressing {0} to {1}'.format(directory, archive)) + + if os.path.exists(archive): + os.remove(archive) + + tarball = None + try: + tarball = tarfile.open(archive, "w") + tarball.add(directory, arcname=os.path.basename(directory)) + finally: + if tarball: + tarball.close() + + +def pre_hdp_select(): + """ + Removes /usr/bin/oozie which is required before running hdp-select + :return: + """ + if os.path.isfile('/usr/bin/oozie'): + os.remove('/usr/bin/oozie') + + +def restore_configuration(): + """ + Restores the configuration backups to their proper locations after an + upgrade has completed. + :return: + """ + Logger.info('Restoring Oozie configuration directory after upgrade...') + directoryMappings = _get_directory_mappings() + + for directory in directoryMappings: + archive = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR, + directoryMappings[directory]) + + if not os.path.isfile(archive): + raise Fail("Unable to restore missing backup archive {0}".format(archive)) + + Logger.info('Extracting {0} to {1}'.format(archive, directory)) + + tarball = None + try: + tarball = tarfile.open(archive, "r") + tarball.extractall(directory) + finally: + if tarball: + tarball.close() + + # cleanup + shutil.rmtree(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR)) + + +def prepare_libext_directory(): + """ + Creates /usr/hdp/current/oozie/libext-customer and recursively sets + 777 permissions on it and its parents. + :return: + """ + import params + + if not os.path.isdir(params.oozie_libext_customer_dir): + os.makedirs(params.oozie_libext_customer_dir, 0o777) + + # ensure that it's rwx for all + os.chmod(params.oozie_libext_customer_dir, 0o777) + + # get all hadooplzo* JAR files + hadoop_lzo_pattern = 'hadoop-lzo*.jar' + files = glob.iglob(os.path.join(params.hadoop_lib_home, hadoop_lzo_pattern)) + if not files: + raise Fail("There are no files at {0} matching {1}".format( + params.hadoop_lib_home, hadoop_lzo_pattern)) + + # copy files into libext + files_copied = False + for file in files: + if os.path.isfile(file): + files_copied = True + Logger.info("Copying {0} to {1}".format(str(file), params.oozie_libext_customer_dir)) + shutil.copy(file, params.oozie_libext_customer_dir) + + if not files_copied: + raise Fail("There are no files at {0} matching {1}".format( + params.hadoop_lib_home, hadoop_lzo_pattern)) + + oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip' + if not os.path.isfile(oozie_ext_zip_file): + raise Fail("Unable to copy {0} because it does not exist".format(oozie_ext_zip_file)) + + Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file, params.oozie_libext_customer_dir)) + shutil.copy(oozie_ext_zip_file, params.oozie_libext_customer_dir) + + +def upgrade_oozie(): + """ + Performs the upgrade of the oozie WAR file and database. + :return: + """ + import params + + # get the kerberos token if necessary to execute commands as oozie + if params.security_enabled: + oozie_principal_with_host = params.oozie_principal.replace("_HOST", params.hostname) + command = format("{kinit_path_local} -kt {oozie_keytab} {oozie_principal_with_host}") + Execute(command, user=params.oozie_user) + + # ensure that HDFS is prepared to receive the new sharelib + command = format("hdfs dfs -chown oozie:hadoop {oozie_hdfs_user_dir}/share") + Execute(command, user=params.oozie_user) + + command = format("hdfs dfs -chmod -R 755 {oozie_hdfs_user_dir}/share") + Execute(command, user=params.oozie_user) + + # upgrade oozie DB + command = format("{oozie_home}/bin/ooziedb.sh upgrade -run") + Execute(command, user=params.oozie_user) + + # prepare the oozie WAR + command = format("{oozie_setup_sh} prepare-war -d {oozie_libext_customer_dir}") + return_code, oozie_output = shell.call(command) + + if return_code != 0 or "New Oozie WAR file with added" not in oozie_output: + message = "Unexpected Oozie WAR preparation output {0}".format(oozie_output) + Logger.error(message) + raise Fail(message) + + # install new sharelib to HDFS + command = format("{oozie_setup_sh} sharelib create -fs {fs_root}") + Execute(command, user=params.oozie_user) + + +def _get_directory_mappings(): + """ + Gets a dictionary of directory to archive name that represents the + directories that need to be backed up and their output tarball archive targets + :return: the dictionary of directory to tarball mappings + """ + import params + + return { params.conf_dir : BACKUP_CONF_ARCHIVE } http://git-wip-us.apache.org/repos/asf/ambari/blob/a8fc971b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py index 59966fd..da61903 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py @@ -20,7 +20,14 @@ limitations under the License. import os from resource_management import * -def oozie_service(action = 'start'): # 'start' or 'stop' +def oozie_service(action = 'start', rolling_restart=False): + """ + Starts or stops the Oozie service + :param action: 'start' or 'stop' + :param rolling_restart: if True, then most of the pre-startup checks are + skipped since a variation of them was performed during the rolling upgrade + :return: + """ import params if params.security_enabled: @@ -43,46 +50,35 @@ def oozie_service(action = 'start'): # 'start' or 'stop' db_connection_check_command = format("{java_home}/bin/java -cp {check_db_connection_jar}:{target} org.apache.ambari.server.DBConnectionVerification '{oozie_jdbc_connection_url}' {oozie_metastore_user_name} {oozie_metastore_user_passwd!p} {jdbc_driver_name}") else: db_connection_check_command = None - - cmd1 = format("cd {oozie_tmp_dir} && {oozie_home}/bin/ooziedb.sh create -sqlfile oozie.sql -run") - cmd2 = format("{kinit_if_needed} {put_shared_lib_to_hdfs_cmd} ; hadoop --config {hadoop_conf_dir} dfs -chmod -R 755 {oozie_hdfs_user_dir}/share") - if not os.path.isfile(params.target) and params.jdbc_driver_name == "org.postgresql.Driver": - print format("ERROR: jdbc file {target} is unavailable. Please, follow next steps:\n" \ - "1) Download postgresql-9.0-801.jdbc4.jar.\n2) Create needed directory: mkdir -p {oozie_home}/libserver/\n" \ - "3) Copy postgresql-9.0-801.jdbc4.jar to newly created dir: cp /path/to/jdbc/postgresql-9.0-801.jdbc4.jar " \ - "{oozie_home}/libserver/\n4) Copy postgresql-9.0-801.jdbc4.jar to libext: cp " \ - "/path/to/jdbc/postgresql-9.0-801.jdbc4.jar {oozie_home}/libext/\n") - exit(1) + if not rolling_restart: + cmd1 = format("cd {oozie_tmp_dir} && {oozie_home}/bin/ooziedb.sh create -sqlfile oozie.sql -run") + cmd2 = format("{kinit_if_needed} {put_shared_lib_to_hdfs_cmd} ; hadoop --config {hadoop_conf_dir} dfs -chmod -R 755 {oozie_hdfs_user_dir}/share") - if db_connection_check_command: - Execute( db_connection_check_command, tries=5, try_sleep=10) - - Execute( cmd1, - user = params.oozie_user, - not_if = no_op_test, - ignore_failures = True - ) - - Execute( cmd2, - user = params.oozie_user, - not_if = format("{kinit_if_needed} hadoop --config {hadoop_conf_dir} dfs -ls /user/oozie/share | awk 'BEGIN {{count=0;}} /share/ {{count++}} END {{if (count > 0) {{exit 0}} else {{exit 1}}}}'"), - path = params.execute_path - ) + if not os.path.isfile(params.target) and params.jdbc_driver_name == "org.postgresql.Driver": + print format("ERROR: jdbc file {target} is unavailable. Please, follow next steps:\n" \ + "1) Download postgresql-9.0-801.jdbc4.jar.\n2) Create needed directory: mkdir -p {oozie_home}/libserver/\n" \ + "3) Copy postgresql-9.0-801.jdbc4.jar to newly created dir: cp /path/to/jdbc/postgresql-9.0-801.jdbc4.jar " \ + "{oozie_home}/libserver/\n4) Copy postgresql-9.0-801.jdbc4.jar to libext: cp " \ + "/path/to/jdbc/postgresql-9.0-801.jdbc4.jar {oozie_home}/libext/\n") + exit(1) + + if db_connection_check_command: + Execute( db_connection_check_command, tries=5, try_sleep=10) + + Execute( cmd1, user = params.oozie_user, not_if = no_op_test, + ignore_failures = True ) + + not_if_command = format("{kinit_if_needed} hadoop --config {hadoop_conf_dir} dfs -ls /user/oozie/share | awk 'BEGIN {{count=0;}} /share/ {{count++}} END {{if (count > 0) {{exit 0}} else {{exit 1}}}}'") + Execute( cmd2, user = params.oozie_user, not_if = not_if_command, + path = params.execute_path ) - Execute( start_cmd, - user = params.oozie_user, - not_if = no_op_test, - ) + Execute( start_cmd, user = params.oozie_user, not_if = no_op_test ) + elif action == 'stop': stop_cmd = format("cd {oozie_tmp_dir} && {oozie_home}/bin/oozie-stop.sh") - Execute(stop_cmd, - only_if = no_op_test, - user = params.oozie_user - ) - File(params.pid_file, - action = "delete", - ) + Execute(stop_cmd, only_if = no_op_test, user = params.oozie_user) + File(params.pid_file, action = "delete") http://git-wip-us.apache.org/repos/asf/ambari/blob/a8fc971b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params.py index 57ea5c1..1274a61 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params.py @@ -18,9 +18,16 @@ limitations under the License. """ -from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions -from resource_management.libraries.functions.default import default -from resource_management import * +from resource_management.core import System +from resource_management.libraries import Script +from resource_management.libraries.functions import default +from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.functions import get_port_from_url +from resource_management.libraries.functions import format +from resource_management.libraries.functions.version import format_hdp_stack_version +from resource_management.libraries.functions.version import compare_versions +from resource_management.libraries.resources import HdfsDirectory + import status_params import itertools import os @@ -29,6 +36,8 @@ import os config = Script.get_config() tmp_dir = Script.get_tmp_dir() +# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade +version = default("/commandParams/version", None) stack_name = default("/hostLevelParams/stack_name", None) stack_version_unformatted = str(config['hostLevelParams']['stack_version']) @@ -38,15 +47,29 @@ hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0: hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin" hadoop_lib_home = "/usr/hdp/current/hadoop-client/lib" - oozie_lib_dir = "/usr/hdp/current/oozie-client/" - oozie_setup_sh = "/usr/hdp/current/oozie-client/bin/oozie-setup.sh" - oozie_webapps_dir = "/usr/hdp/current/oozie-client/oozie-server/webapps" - oozie_webapps_conf_dir = "/usr/hdp/current/oozie-client/oozie-server/conf" - oozie_libext_dir = "/usr/hdp/current/oozie-client/libext" - oozie_server_dir = "/usr/hdp/current/oozie-client/oozie-server" - oozie_shared_lib = "/usr/hdp/current/oozie-client/share" - oozie_home = "/usr/hdp/current/oozie-client" - oozie_bin_dir = "/usr/hdp/current/oozie-client/bin" + + # if this is a server action, then use the server binaries; smoke tests + # use the client binaries + server_role_dir_mapping = { 'OOZIE_SERVER' : 'oozie-server', + 'OOZIE_SERVICE_CHECK' : 'oozie-client' } + + command_role = default("/role", "") + if command_role not in server_role_dir_mapping: + command_role = 'OOZIE_SERVICE_CHECK' + + oozie_root = server_role_dir_mapping[command_role] + + # using the correct oozie root dir, format the correct location + oozie_lib_dir = format("/usr/hdp/current/{oozie_root}/") + oozie_setup_sh = format("/usr/hdp/current/{oozie_root}/bin/oozie-setup.sh") + oozie_webapps_dir = format("/usr/hdp/current/{oozie_root}/oozie-server/webapps") + oozie_webapps_conf_dir = format("/usr/hdp/current/{oozie_root}/oozie-server/conf") + oozie_libext_dir = format("/usr/hdp/current/{oozie_root}/libext") + oozie_libext_customer_dir = format("/usr/hdp/current/{oozie_root}/libext-customer") + oozie_server_dir = format("/usr/hdp/current/{oozie_root}/oozie-server") + oozie_shared_lib = format("/usr/hdp/current/{oozie_root}/share") + oozie_home = format("/usr/hdp/current/{oozie_root}") + oozie_bin_dir = format("/usr/hdp/current/{oozie_root}/bin") falcon_home = '/usr/hdp/current/falcon-client' else: hadoop_bin_dir = "/usr/bin" @@ -86,7 +109,7 @@ security_enabled = config['configurations']['cluster-env']['security_enabled'] oozie_heapsize = config['configurations']['oozie-env']['oozie_heapsize'] oozie_permsize = config['configurations']['oozie-env']['oozie_permsize'] -kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) +kinit_path_local = get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) oozie_service_keytab = config['configurations']['oozie-site']['oozie.service.HadoopAccessorService.keytab.file'] oozie_principal = config['configurations']['oozie-site']['oozie.service.HadoopAccessorService.kerberos.principal'] smokeuser_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] http://git-wip-us.apache.org/repos/asf/ambari/blob/a8fc971b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml index ed6b50c..6ecb882 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml @@ -76,6 +76,14 @@ </service> </group> + <group name="OOZIE" title="Oozie"> + <skippable>true</skippable> + <service name="OOZIE"> + <component>OOZIE_SERVER</component> + <component>OOZIE_CLIENT</component> + </service> + </group> + <group name="FALCON" title="Falcon"> <skippable>true</skippable> <service name="FALCON"> @@ -314,7 +322,7 @@ <component name="HIVE_SERVER"> <pre-upgrade> <task xsi:type="manual"> - <message>The HiveServer port will now change to 10010. Ensure that this port is available on each HiveServer instance.</message> + <message>The HiveServer port will now change to 10010. You can use "netstat -anp | grep 10010" to determine if the port is available on each HiveServer host. If the port is not available, the process using it must be terminated.</message> </task> <task xsi:type="configure"> @@ -326,7 +334,7 @@ <pre-downgrade> <task xsi:type="manual"> - <message>The HiveServer port will now change to 10000. Ensure that this port is available on each HiveServer instance.</message> + <message>The HiveServer port will now change to 10000. You can use "netstat -anp | grep 10000" to determine if the port is available on each HiveServer host. If the port is not available, the process using it must be terminated.</message> </task> <task xsi:type="configure"> @@ -354,6 +362,24 @@ </component> </service> + <service name="OOZIE"> + <component name="OOZIE_SERVER"> + <pre-upgrade> + <task xsi:type="manual"> + <message>Backup the Oozie Server database.</message> + </task> + </pre-upgrade> + <upgrade> + <task xsi:type="restart" /> + </upgrade> + </component> + <component name="OOZIE_CLIENT"> + <upgrade> + <task xsi:type="restart" /> + </upgrade> + </component> + </service> + <service name="FALCON"> <component name="FALCON_SERVER"> <upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/a8fc971b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py index 4324040..739780c 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py @@ -17,12 +17,17 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ''' + from mock.mock import MagicMock, call, patch from stacks.utils.RMFTestCase import * +from resource_management.core import shell +from resource_management.core.exceptions import Fail + class TestOozieServer(RMFTestCase): COMMON_SERVICES_PACKAGE_DIR = "OOZIE/4.0.0.2.0/package" STACK_VERSION = "2.0.6" + UPGRADE_STACK_VERSION = "2.2" def test_configure_default(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py", @@ -593,3 +598,86 @@ class TestOozieServer(RMFTestCase): target = RMFTestCase.TARGET_COMMON_SERVICES ) put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"}) + + + @patch("tarfile.open") + @patch("os.path.isdir") + @patch("os.path.exists") + @patch("os.path.isfile") + @patch("os.remove") + @patch("os.chmod") + @patch("shutil.rmtree", new = MagicMock()) + @patch("glob.iglob", new = MagicMock(return_value=["/usr/hdp/2.2.1.0-2187/hadoop/lib/hadoop-lzo-0.6.0.2.2.1.0-2187.jar"])) + @patch("shutil.copy") + @patch.object(shell, "call") + def test_upgrade(self, call_mock, shutil_copy_mock, chmod_mock, remove_mock, + isfile_mock, exists_mock, isdir_mock, tarfile_open_mock): + + isdir_mock.return_value = True + exists_mock.side_effect = [False,False,True] + isfile_mock.return_value = True + + prepare_war_stdout = """INFO: Adding extension: libext/mysql-connector-java.jar + New Oozie WAR file with added 'JARs' at /var/lib/oozie/oozie-server/webapps/oozie.war""" + + call_mock.return_value = (0, prepare_war_stdout) + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py", + classname = "OozieServer", command = "pre_rolling_restart", config_file = "oozie-upgrade.json", + hdp_stack_version = self.UPGRADE_STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES ) + + # 2 calls to tarfile.open (1 directories, read + write) + self.assertTrue(tarfile_open_mock.called) + self.assertEqual(tarfile_open_mock.call_count,2) + + self.assertTrue(chmod_mock.called) + self.assertEqual(chmod_mock.call_count,1) + chmod_mock.assert_called_once_with('/usr/hdp/current/oozie-server/libext-customer', 511) + + self.assertTrue(isfile_mock.called) + self.assertEqual(isfile_mock.call_count,4) + isfile_mock.assert_called_with('/usr/share/HDP-oozie/ext-2.2.zip') + + self.assertTrue(remove_mock.called) + self.assertEqual(remove_mock.call_count,1) + remove_mock.assert_called_with('/usr/bin/oozie') + + self.assertResourceCalled('Execute', 'hdp-select set oozie-server 2.2.1.0-2135') + self.assertResourceCalled('Execute', 'hdfs dfs -chown oozie:hadoop /user/oozie/share', user='oozie') + self.assertResourceCalled('Execute', 'hdfs dfs -chmod -R 755 /user/oozie/share', user='oozie') + self.assertResourceCalled('Execute', '/usr/hdp/current/oozie-server/bin/ooziedb.sh upgrade -run', user='oozie') + self.assertResourceCalled('Execute', '/usr/hdp/current/oozie-server/bin/oozie-setup.sh sharelib create -fs hdfs://c6401.ambari.apache.org:8020', user='oozie') + + self.assertNoMoreResources() + + @patch("tarfile.open") + @patch("os.path.isdir") + @patch("os.path.exists") + @patch("os.path.isfile") + @patch("os.remove") + @patch("os.chmod") + @patch("shutil.rmtree", new = MagicMock()) + @patch("glob.iglob", new = MagicMock(return_value=["/usr/hdp/2.2.1.0-2187/hadoop/lib/hadoop-lzo-0.6.0.2.2.1.0-2187.jar"])) + @patch("shutil.copy") + @patch.object(shell, "call") + def test_upgrade_failed_prepare_war(self, call_mock, shutil_copy_mock, chmod_mock, remove_mock, + isfile_mock, exists_mock, isdir_mock, tarfile_open_mock): + + isdir_mock.return_value = True + exists_mock.side_effect = [False,False,True] + isfile_mock.return_value = True + + call_mock.return_value = (0, 'Whoops, you messed up the WAR.') + + try: + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py", + classname = "OozieServer", command = "pre_rolling_restart", config_file = "oozie-upgrade.json", + hdp_stack_version = self.UPGRADE_STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES ) + + self.fail("An invalid WAR preparation should have caused an error") + except Fail,f: + pass + +
