Repository: ambari Updated Branches: refs/heads/trunk 167baf39e -> b7e1bcebf
AMBARI-11630 - Falcon Server Seems To Have Hadoop 2.2 Paths After Upgrade (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b7e1bceb Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b7e1bceb Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b7e1bceb Branch: refs/heads/trunk Commit: b7e1bcebf8244a63ffc594928dbb278370261c68 Parents: 167baf3 Author: Jonathan Hurley <[email protected]> Authored: Tue Jun 2 19:48:27 2015 -0400 Committer: Jonathan Hurley <[email protected]> Committed: Tue Jun 2 20:53:30 2015 -0400 ---------------------------------------------------------------------- .../FALCON/0.5.0.2.1/package/scripts/falcon.py | 244 ++++++++++--------- .../0.5.0.2.1/package/scripts/falcon_server.py | 21 +- .../0.5.0.2.1/package/scripts/params_linux.py | 1 + .../stacks/2.1/FALCON/test_falcon_server.py | 54 ++-- 4 files changed, 178 insertions(+), 142 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b7e1bceb/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py index b9821bf..db93e7d 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py @@ -17,177 +17,195 @@ limitations under the License. """ -from resource_management import * import os.path + +from resource_management.core.environment import Environment +from resource_management.core.source import InlineTemplate +from resource_management.core.source import Template +from resource_management.core.resources import Execute +from resource_management.core.resources.service import Service +from resource_management.core.resources.service import ServiceConfig +from resource_management.core.resources.system import Directory +from resource_management.core.resources.system import File +from resource_management.libraries.script import Script +from resource_management.libraries.resources import PropertiesFile +from resource_management.libraries.functions import format + from ambari_commons import OSConst from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl -@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +@OsFamilyFuncImpl(os_family = OsFamilyImpl.DEFAULT) def falcon(type, action = None): import params + if action == 'config': Directory(params.falcon_pid_dir, - owner=params.falcon_user, - recursive=True - ) + owner = params.falcon_user, + recursive = True) + Directory(params.falcon_log_dir, - owner=params.falcon_user, - recursive=True - ) + owner = params.falcon_user, + recursive = True) + Directory(params.falcon_webapp_dir, - owner=params.falcon_user, - recursive=True - ) + owner = params.falcon_user, + recursive = True) + Directory(params.falcon_home, - owner=params.falcon_user, - recursive=True - ) + owner = params.falcon_user, + recursive = True) + Directory(params.etc_prefix_dir, - mode=0755, - recursive=True - ) + mode = 0755, + recursive = True) + Directory(params.falcon_conf_dir, - owner=params.falcon_user, - recursive=True - ) + owner = params.falcon_user, + recursive = True) + File(params.falcon_conf_dir + '/falcon-env.sh', - content=InlineTemplate(params.falcon_env_sh_template), - owner=params.falcon_user - ) + content = InlineTemplate(params.falcon_env_sh_template), + owner = params.falcon_user) + File(params.falcon_conf_dir + '/client.properties', - content=Template('client.properties.j2'), - mode=0644, - owner=params.falcon_user - ) + content = Template('client.properties.j2'), + mode = 0644, + owner = params.falcon_user) + PropertiesFile(params.falcon_conf_dir + '/runtime.properties', - properties=params.falcon_runtime_properties, - mode=0644, - owner=params.falcon_user - ) + properties = params.falcon_runtime_properties, + mode = 0644, + owner = params.falcon_user) + PropertiesFile(params.falcon_conf_dir + '/startup.properties', - properties=params.falcon_startup_properties, - mode=0644, - owner=params.falcon_user - ) + properties = params.falcon_startup_properties, + mode = 0644, + owner = params.falcon_user) if params.falcon_graph_storage_directory: Directory(params.falcon_graph_storage_directory, - owner=params.falcon_user, - group=params.user_group, - mode=0775, - recursive=True, - cd_access="a" - ) + owner = params.falcon_user, + group = params.user_group, + mode = 0775, + recursive = True, + cd_access = "a") if params.falcon_graph_serialize_path: Directory(params.falcon_graph_serialize_path, - owner=params.falcon_user, - group=params.user_group, - mode=0775, - recursive=True, - cd_access="a" - ) + owner = params.falcon_user, + group = params.user_group, + mode = 0775, + recursive = True, + cd_access = "a") if type == 'server': if action == 'config': if params.store_uri[0:4] == "hdfs": params.HdfsResource(params.store_uri, - type="directory", - action="create_on_execute", - owner=params.falcon_user, - mode=0755 - ) + type = "directory", + action = "create_on_execute", + owner = params.falcon_user, + mode = 0755) elif params.store_uri[0:4] == "file": Directory(params.store_uri[7:], - owner=params.falcon_user, - recursive=True - ) + owner = params.falcon_user, + recursive = True) + + # TODO change to proper mode params.HdfsResource(params.flacon_apps_dir, - type="directory", - action="create_on_execute", - owner=params.falcon_user, - mode=0777 #TODO change to proper mode - ) + type = "directory", + action = "create_on_execute", + owner = params.falcon_user, + mode = 0777) + if params.falcon_store_uri[0:4] == "hdfs": params.HdfsResource(params.falcon_store_uri, - type="directory", - action="create_on_execute", - owner=params.falcon_user, - mode=0755 - ) + type = "directory", + action = "create_on_execute", + owner = params.falcon_user, + mode = 0755) elif params.falcon_store_uri[0:4] == "file": Directory(params.falcon_store_uri[7:], - owner=params.falcon_user, - recursive=True - ) + owner = params.falcon_user, + recursive = True) + if params.supports_hive_dr: params.HdfsResource(params.dfs_data_mirroring_dir, - type="directory", - action="create_on_execute", - owner=params.falcon_user, - group=params.user_group, - recursive_chown = True, - recursive_chmod = True, - mode=0770, - source=params.local_data_mirroring_dir - ) - params.HdfsResource(None, action="execute") + type = "directory", + action = "create_on_execute", + owner = params.falcon_user, + group = params.user_group, + recursive_chown = True, + recursive_chmod = True, + mode = 0770, + source = params.local_data_mirroring_dir) + + params.HdfsResource(None, action = "execute") Directory(params.falcon_local_dir, - owner=params.falcon_user, - recursive=True, - cd_access="a" - ) + owner = params.falcon_user, + recursive = True, + cd_access = "a") + if params.falcon_embeddedmq_enabled == True: - Directory(os.path.abspath(os.path.join(params.falcon_embeddedmq_data, "..")), - owner=params.falcon_user, - recursive=True - ) + Directory( + os.path.abspath(os.path.join(params.falcon_embeddedmq_data, "..")), + owner = params.falcon_user, + recursive = True) + Directory(params.falcon_embeddedmq_data, - owner=params.falcon_user, - recursive=True - ) + owner = params.falcon_user, + recursive = True) + + # although Falcon's falcon-config.sh will use 'which hadoop' to figure + # this out, in an upgraded cluster, it's possible that 'which hadoop' + # still points to older binaries; it's safer to just pass in the + # hadoop home directory to use + environment_dictionary = { "HADOOP_HOME" : params.hadoop_home_dir } if action == 'start': Execute(format('{falcon_home}/bin/falcon-start -port {falcon_port}'), - user=params.falcon_user, - path=params.hadoop_bin_dir - ) + user = params.falcon_user, + path = params.hadoop_bin_dir, + environment=environment_dictionary) + if action == 'stop': Execute(format('{falcon_home}/bin/falcon-stop'), - user=params.falcon_user, - path=params.hadoop_bin_dir - ) - File(params.server_pid_file, - action='delete' - ) - -@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) + user = params.falcon_user, + path = params.hadoop_bin_dir, + environment=environment_dictionary) + + File(params.server_pid_file, action = 'delete') + + +@OsFamilyFuncImpl(os_family = OSConst.WINSRV_FAMILY) def falcon(type, action = None): import params + if action == 'config': env = Environment.get_instance() # These 2 parameters are used in ../templates/client.properties.j2 env.config.params["falcon_host"] = params.falcon_host env.config.params["falcon_port"] = params.falcon_port File(os.path.join(params.falcon_conf_dir, 'falcon-env.sh'), - content=InlineTemplate(params.falcon_env_sh_template) - ) + content = InlineTemplate(params.falcon_env_sh_template)) + File(os.path.join(params.falcon_conf_dir, 'client.properties'), - content=Template('client.properties.j2') - ) + content = Template('client.properties.j2')) + PropertiesFile(os.path.join(params.falcon_conf_dir, 'runtime.properties'), - properties=params.falcon_runtime_properties - ) + properties = params.falcon_runtime_properties) + PropertiesFile(os.path.join(params.falcon_conf_dir, 'startup.properties'), - properties=params.falcon_startup_properties - ) + properties = params.falcon_startup_properties) if type == 'server': ServiceConfig(params.falcon_win_service_name, - action="change_user", - username=params.falcon_user, - password = Script.get_password(params.falcon_user)) + action = "change_user", + username = params.falcon_user, + password = Script.get_password(params.falcon_user)) + if action == 'start': - Service(params.falcon_win_service_name, action="start") + Service(params.falcon_win_service_name, action = "start") + if action == 'stop': - Service(params.falcon_win_service_name, action="stop") \ No newline at end of file + Service(params.falcon_win_service_name, action = "stop") \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/b7e1bceb/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py index 2598ec5..3436e5b 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py @@ -19,13 +19,17 @@ limitations under the License. import falcon_server_upgrade -from resource_management import * +from resource_management.core.logger import Logger +from resource_management.libraries.script import Script from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import hdp_select -from resource_management.libraries.functions.version import * -from resource_management.libraries.functions.security_commons import build_expectations, \ - cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ - FILE_TYPE_PROPERTIES +from resource_management.libraries.functions import check_process_status +from resource_management.libraries.functions.security_commons import build_expectations +from resource_management.libraries.functions.security_commons import cached_kinit_executor +from resource_management.libraries.functions.security_commons import get_params_from_filesystem +from resource_management.libraries.functions.security_commons import validate_security_config_properties +from resource_management.libraries.functions.security_commons import FILE_TYPE_PROPERTIES + from falcon import falcon from ambari_commons import OSConst from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl @@ -46,6 +50,7 @@ class FalconServer(Script): import params env.set_params(params) falcon('server', action='stop') + # if performing an upgrade, backup some directories after stopping falcon if rolling_restart: falcon_server_upgrade.post_stop_backup() @@ -71,7 +76,7 @@ class FalconServerLinux(FalconServer): # this function should not execute if the version can't be determined or # is not at least HDP 2.2.0.0 - if not params.version or compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') < 0: + if Script.is_hdp_stack_less_than("2.2"): return Logger.info("Executing Falcon Server Rolling Upgrade pre-restart") @@ -144,11 +149,15 @@ class FalconServerWindows(FalconServer): def install(self, env): import params + from resource_management.libraries.functions.windows_service_utils import check_windows_service_exists + if not check_windows_service_exists(params.falcon_win_service_name): self.install_packages(env) def status(self, env): import status_params + from resource_management.libraries.functions.windows_service_utils import check_windows_service_status + check_windows_service_status(status_params.falcon_win_service_name) if __name__ == "__main__": http://git-wip-us.apache.org/repos/asf/ambari/blob/b7e1bceb/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py index 9e79daf..d79026a 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py @@ -38,6 +38,7 @@ hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) etc_prefix_dir = "/etc/falcon" # hadoop params +hadoop_home_dir = hdp_select.get_hadoop_dir("home") hadoop_bin_dir = hdp_select.get_hadoop_dir("bin") if Script.is_hdp_stack_greater_or_equal("2.2"): http://git-wip-us.apache.org/repos/asf/ambari/blob/b7e1bceb/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py index 326bdc8..7c79bad 100644 --- a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py +++ b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py @@ -33,34 +33,37 @@ class TestFalconServer(RMFTestCase): def test_start_default(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py", - classname="FalconServer", - command="start", - config_file="default.json", - hdp_stack_version = self.STACK_VERSION, - target = RMFTestCase.TARGET_COMMON_SERVICES - ) + classname="FalconServer", + command="start", + config_file="default.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES) + self.assert_configure_default() + self.assertResourceCalled('Execute', '/usr/lib/falcon/bin/falcon-start -port 15000', - path = ['/usr/bin'], - user = 'falcon', - ) + path = ['/usr/bin'], + user = 'falcon', + environment = {'HADOOP_HOME': '/usr/lib/hadoop'}) + self.assertNoMoreResources() def test_stop_default(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py", - classname="FalconServer", - command="stop", - config_file="default.json", - hdp_stack_version = self.STACK_VERSION, - target = RMFTestCase.TARGET_COMMON_SERVICES - ) + classname="FalconServer", + command="stop", + config_file="default.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES) + self.assertResourceCalled('Execute', '/usr/lib/falcon/bin/falcon-stop', - path = ['/usr/bin'], - user = 'falcon', - ) + path = ['/usr/bin'], + user = 'falcon', + environment = {'HADOOP_HOME': '/usr/lib/hadoop'}) + self.assertResourceCalled('File', '/var/run/falcon/falcon.pid', - action = ['delete'], - ) + action = ['delete']) + self.assertNoMoreResources() def test_configure_default(self): @@ -198,7 +201,8 @@ class TestFalconServer(RMFTestCase): self.assertResourceCalled('Execute', '/usr/hdp/current/falcon-server/bin/falcon-stop', - path = ['/usr/hdp/current/hadoop-client/bin'], user='falcon') + path = ['/usr/hdp/current/hadoop-client/bin'], user='falcon', + environment = {'HADOOP_HOME': '/usr/hdp/current/hadoop-client'}) self.assertResourceCalled('File', '/var/run/falcon/falcon.pid', action = ['delete']) @@ -322,10 +326,12 @@ class TestFalconServer(RMFTestCase): @patch.object(tarfile, 'open') @patch.object(shutil, 'rmtree') def test_pre_rolling_restart(self, tarfile_open_mock, rmtree_mock): - config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json" + config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/falcon-upgrade.json" tarfile_open_mock.return_value = MagicMock() + with open(config_file, "r") as f: json_content = json.load(f) + version = '2.2.1.0-3242' json_content['commandParams']['version'] = version self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py", @@ -345,9 +351,11 @@ class TestFalconServer(RMFTestCase): @patch.object(shutil, 'rmtree') @patch("resource_management.core.shell.call") def test_pre_rolling_restart_23(self, tarfile_open_mock, rmtree_mock, call_mock): - config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json" + config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/falcon-upgrade.json" + with open(config_file, "r") as f: json_content = json.load(f) + version = '2.3.0.0-1234' json_content['commandParams']['version'] = version
