This is an automated email from the ASF dual-hosted git repository. jgolieb pushed a commit to branch branch-feature-AMBARI-14714-mpack-advisor in repository https://gitbox.apache.org/repos/asf/ambari.git
commit 859d08cb40b0af7118ed9e6a2d3284af03f9c148 Author: sduan <[email protected]> AuthorDate: Thu May 31 12:02:59 2018 -0700 [AMBARI-23993] Mpack Instance Manager should create pid dir and log dir for each instance (#1424) * AMBARI-23993: Mpack Instance Manager should create pid dir and log dir for each instance * AMBARI-23993: Mpack Instance Manager should create pid dir and log dir for each instance --- .../libraries/functions/mpack_manager_helper.py | 27 ++++++ .../python/instance_manager/instance_manager.py | 74 +++++++++++--- .../instance_manager/mpack-instance-manager.py | 18 +++- .../instance_manager/test_instance_manager.py | 108 +++++++++++++++++++++ 4 files changed, 212 insertions(+), 15 deletions(-) diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/mpack_manager_helper.py b/ambari-common/src/main/python/resource_management/libraries/functions/mpack_manager_helper.py index e99ff27..1bae115 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/mpack_manager_helper.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/mpack_manager_helper.py @@ -23,6 +23,8 @@ import os from instance_manager import create_mpack, set_mpack_instance, get_conf_dir, list_instances CONFIG_DIR_KEY_NAME = 'config_dir' +LOG_DIR_KEY_NAME = 'log_dir' +RUN_DIR_KEY_NAME = 'run_dir' PATH_KEY_NAME = 'path' COMPONENTS_PLURAL_KEY_NAME = 'components' COMPONENT_INSTANCES_PLURAL_KEY_NAME = 'component-instances' @@ -41,6 +43,31 @@ def get_component_conf_path(mpack_name, instance_name, module_name, components_i return conf_json[COMPONENTS_PLURAL_KEY_NAME][components_instance_type.lower()][COMPONENT_INSTANCES_PLURAL_KEY_NAME][ component_instance_name][CONFIG_DIR_KEY_NAME] +def get_component_log_path(mpack_name, instance_name, module_name, components_instance_type, + subgroup_name='default', component_instance_name='default'): + """ + :returns the single string that contains the path to the log folder of given component instance + :raises ValueError if the parameters doesn't match the mpack or instances structure + """ + + log_json = get_log_dir(mpack_name, instance_name, subgroup_name, module_name, + {components_instance_type: [component_instance_name]}) + + return log_json[COMPONENTS_PLURAL_KEY_NAME][components_instance_type.lower()][COMPONENT_INSTANCES_PLURAL_KEY_NAME][ + component_instance_name][LOG_DIR_KEY_NAME] + +def get_component_rundir_path(mpack_name, instance_name, module_name, components_instance_type, + subgroup_name='default', component_instance_name='default'): + """ + :returns the single string that contains the path to the rundir folder of given component instance + :raises ValueError if the parameters doesn't match the mpack or instances structure + """ + + run_json = get_run_dir(mpack_name, instance_name, subgroup_name, module_name, + {components_instance_type: [component_instance_name]}) + + return run_json[COMPONENTS_PLURAL_KEY_NAME][components_instance_type.lower()][COMPONENT_INSTANCES_PLURAL_KEY_NAME][ + component_instance_name][RUN_DIR_KEY_NAME] def get_component_target_path(mpack_name, instance_name, module_name, components_instance_type, subgroup_name='default', component_instance_name='default'): diff --git a/mpack-instance-manager/src/main/python/instance_manager/instance_manager.py b/mpack-instance-manager/src/main/python/instance_manager/instance_manager.py index 39e5ea6..0a9ed83 100644 --- a/mpack-instance-manager/src/main/python/instance_manager/instance_manager.py +++ b/mpack-instance-manager/src/main/python/instance_manager/instance_manager.py @@ -17,7 +17,7 @@ limitations under the License. """ -__all__ = ["create_mpack", "set_mpack_instance", "get_conf_dir", "list_instances"] +__all__ = ["create_mpack", "set_mpack_instance", "get_conf_dir", "get_log_dir", "get_run_dir", "list_instances"] import sys import os @@ -26,6 +26,8 @@ import json MPACK_JSON_FILE_NAME = 'mpack.json' CURRENT_SOFTLINK_NAME = 'current' CONFIGS_DIRECTORY_NAME = 'conf' +LOG_DIRECTORY_NAME = 'log' +RUN_DIRECTORY_NAME = 'run' ROOT_FOLDER_PATH = "/usr/hwx/" @@ -110,6 +112,46 @@ def get_conf_dir(mpack=None, mpack_instance=None, subgroup_name=DEFAULT_SUBGROUP return build_granular_json_with_filtering(mpack, mpack_instance, subgroup_name, module_name, components_map, output_conf_dir=True) +def get_log_dir(mpack=None, mpack_instance=None, subgroup_name=DEFAULT_SUBGROUP_NAME, module_name=None, + components_map=None): + """ + Use case: retrieve log directory paths for a given component instances based on the granularity specified + ranging from: mpack, mpack-instance, subgroup-name, module-name and map of component instance + AND with a filtering on each level + + Granularity works only while names for all consecutive levels are specified. + Note that subgroup has default value of 'default' + Levels: mpack/instance/subgroup/module + E.g If only mpack and subgroup names are specified, the granularity will work only on mpack level, + though the subgroup fitler will be applied. But if the instance name is specified also, than only granular output + of subgroup will be returned. + + Components are provided as map with key as 'component type' and value as 'list of individual component instances + names' OR empty map for all component instances present + """ + return build_granular_json_with_filtering(mpack, mpack_instance, subgroup_name, module_name, components_map, + output_log_dir=True) + +def get_run_dir(mpack=None, mpack_instance=None, subgroup_name=DEFAULT_SUBGROUP_NAME, module_name=None, + components_map=None): + """ + Use case: retrieve run directory paths for a given component instances based on the granularity specified + ranging from: mpack, mpack-instance, subgroup-name, module-name and map of component instance + AND with a filtering on each level + + Granularity works only while names for all consecutive levels are specified. + Note that subgroup has default value of 'default' + Levels: mpack/instance/subgroup/module + E.g If only mpack and subgroup names are specified, the granularity will work only on mpack level, + though the subgroup fitler will be applied. But if the instance name is specified also, than only granular output + of subgroup will be returned. + + Components are provided as map with key as 'component type' and value as 'list of individual component instances + names' OR empty map for all component instances present + """ + return build_granular_json_with_filtering(mpack, mpack_instance, subgroup_name, module_name, components_map, + output_run_dir=True) + def list_instances(mpack=None, mpack_instance=None, subgroup_name=DEFAULT_SUBGROUP_NAME, module_name=None, components_map=None): @@ -134,7 +176,7 @@ def list_instances(mpack=None, mpack_instance=None, subgroup_name=DEFAULT_SUBGRO def build_granular_json_with_filtering(mpack_name_filter, instance_name_filter, subgroup_name_filter, module_name_filter, components_name_filter_map, output_conf_dir=False, - output_path=False): + output_log_dir=False, output_run_dir=False, output_path=False): """ Builds the json that contains all instances specified in filters or all instances if filters are not specified. The level of granularity depends on the consecutive levels of specified filters @@ -160,7 +202,7 @@ def build_granular_json_with_filtering(mpack_name_filter, instance_name_filter, format(mpack_name_filter, instance_name_filter, subgroup_name_filter, module_name_filter, components_name_filter_map)) - full_json_output = build_json_output(instances, output_conf_dir=output_conf_dir, output_path=output_path) + full_json_output = build_json_output(instances, output_conf_dir=output_conf_dir, output_log_dir=output_log_dir, output_run_dir=output_run_dir, output_path=output_path) granular_json_output = build_granular_output(full_json_output, mpack_name_filter, instance_name_filter, subgroup_name_filter, @@ -190,14 +232,14 @@ def build_granular_output(json_output, mpack_name_filter, instance_name_filter, return json_output -def build_json_output_from_instances_dict(instances_dict, plural_name, output_conf_dir, output_path): +def build_json_output_from_instances_dict(instances_dict, plural_name, output_conf_dir, output_log_dir, output_run_dir, output_path): """ Build the json from the dictionary of Instance objects. The plural_name is used to form the upper level of the json output. """ result = {} for instance_name in instances_dict: - result[instance_name] = instances_dict[instance_name].build_json_output(output_conf_dir, output_path) + result[instance_name] = instances_dict[instance_name].build_json_output(output_conf_dir, output_log_dir, output_run_dir, output_path) return {plural_name: result} @@ -224,11 +266,11 @@ def get_module_meta_mpack(path, module_name): return MetaMpack.parse_mpack(os.path.dirname(current_target)) -def build_json_output(instances, output_conf_dir=False, output_path=False): +def build_json_output(instances, output_conf_dir=False, output_log_dir=False, output_run_dir=False, output_path=False): result = {} for mpack_name in instances.keys(): result[mpack_name] = build_json_output_from_instances_dict(instances[mpack_name], MpackInstance.plural_name, - output_conf_dir, output_path) + output_conf_dir, output_log_dir, output_run_dir, output_path) return {'mpacks': result} @@ -365,7 +407,7 @@ class MetaMpack: class Instance: - def build_json_output(self, output_conf_dir, output_path): + def build_json_output(self, output_conf_dir, output_log_dir, output_run_dir, output_path): raise NotImplementedError("Should have implemented this") @@ -377,11 +419,11 @@ class MpackInstance(Instance): self.instance_name = instance_name self.groups_dict = groups_dict - def build_json_output(self, output_conf_dir, output_path): + def build_json_output(self, output_conf_dir, output_log_dir, output_run_dir, output_path): result = {} for group in self.groups_dict.keys(): result[group] = build_json_output_from_instances_dict(self.groups_dict[group], ModuleInstance.plural_name, - output_conf_dir, output_path) + output_conf_dir, output_log_dir, output_run_dir, output_path) return {"subgroups": result, 'name': self.instance_name} def set_new_version(self, mpack_name, mpack_version): @@ -536,12 +578,12 @@ class ModuleInstance(Instance): print("\nSetting new version for component : " + component_instance.component_path) component_instance.set_new_version(mpack_name, mpack_version, component_type) - def build_json_output(self, output_conf_dir, output_path): + def build_json_output(self, output_conf_dir, output_log_dir, output_run_dir, output_path): result = {} for component_type in self.components_map.keys(): result[component_type] = build_json_output_from_instances_dict(self.components_map[component_type], ComponentInstance.plural_name, - output_conf_dir, output_path) + output_conf_dir, output_log_dir, output_run_dir, output_path) result = {'components': result} result['category'] = self.category result['name'] = self.module_name @@ -631,12 +673,18 @@ class ComponentInstance(Instance): os.symlink(mpack_path, os.path.join(component_path, CURRENT_SOFTLINK_NAME)) os.makedirs(os.path.join(component_path, CONFIGS_DIRECTORY_NAME)) + os.makedirs(os.path.join(component_path, LOG_DIRECTORY_NAME)) + os.makedirs(os.path.join(component_path, RUN_DIRECTORY_NAME)) print "\n Created " + component_path - def build_json_output(self, output_conf_dir, output_path): + def build_json_output(self, output_conf_dir, output_log_dir, output_run_dir, output_path): result = {'name': self.name} if output_conf_dir: result['config_dir'] = os.path.join(self.component_path, CONFIGS_DIRECTORY_NAME) + if output_log_dir: + result['log_dir'] = os.path.join(self.component_path, LOG_DIRECTORY_NAME) + if output_run_dir: + result['run_dir'] = os.path.join(self.component_path, RUN_DIRECTORY_NAME) if output_path: result['path'] = self.path_exec return result diff --git a/mpack-instance-manager/src/main/python/instance_manager/mpack-instance-manager.py b/mpack-instance-manager/src/main/python/instance_manager/mpack-instance-manager.py index f9458b1..4332d25 100644 --- a/mpack-instance-manager/src/main/python/instance_manager/mpack-instance-manager.py +++ b/mpack-instance-manager/src/main/python/instance_manager/mpack-instance-manager.py @@ -26,6 +26,8 @@ import ast CREATE_MPACK_INSTANCE_ACTION = 'create-mpack-instance' SET_MPACK_INSTANCE_ACTION = 'set-mpack-instance' GET_CONF_DIR_ACTION = 'get-conf-dir' +GET_LOG_DIR_ACTION = 'get-log-dir' +GET_RUN_DIR_ACTION = 'get-run-dir' LIST_INSTANCES_ACTION = 'list-instances' @@ -34,6 +36,8 @@ def init_action_parser(action, parser): CREATE_MPACK_INSTANCE_ACTION: init_create_parser_options, SET_MPACK_INSTANCE_ACTION: init_set_parser_options, GET_CONF_DIR_ACTION: init_get_parser_options, + GET_LOG_DIR_ACTION: init_get_parser_options, + GET_RUN_DIR_ACTION: init_get_parser_options, LIST_INSTANCES_ACTION: init_get_parser_options } try: @@ -152,6 +156,16 @@ def main(options, args): subgroup_name=options.subgroup_name, module_name=options.module_name, components_map=parsed_components_map) + elif action == GET_LOG_DIR_ACTION: + print get_log_dir(mpack=options.mpack, mpack_instance=options.mpack_instance, + subgroup_name=options.subgroup_name, module_name=options.module_name, + components_map=parsed_components_map) + + elif action == GET_RUN_DIR_ACTION: + print get_run_dir(mpack=options.mpack, mpack_instance=options.mpack_instance, + subgroup_name=options.subgroup_name, module_name=options.module_name, + components_map=parsed_components_map) + elif action == LIST_INSTANCES_ACTION: print list_instances(mpack=options.mpack, mpack_instance=options.mpack_instance, subgroup_name=options.subgroup_name, module_name=options.module_name, @@ -161,10 +175,10 @@ def main(options, args): if __name__ == "__main__": if len(sys.argv) < 2: print( - "Missing the command. Possible options are: create-mpack-instance|set-mpack-instance|get-conf-dir|list-instances") + "Missing the command. Possible options are: create-mpack-instance|set-mpack-instance|get-conf-dir|get-log-dir|get-run-dir|list-instances") sys.exit(1) - usage = "Usage: %prog create-mpack-instance|set-mpack-instance|get-conf-dir|list-instances [options]" + usage = "Usage: %prog create-mpack-instance|set-mpack-instance|get-conf-dir|get-log-dir|get-run-dir|list-instances [options]" parser = optparse.OptionParser(usage=usage) action = sys.argv[1] init_action_parser(action, parser) diff --git a/mpack-instance-manager/src/test/python/instance_manager/test_instance_manager.py b/mpack-instance-manager/src/test/python/instance_manager/test_instance_manager.py index 5d359a4..db5c521 100644 --- a/mpack-instance-manager/src/test/python/instance_manager/test_instance_manager.py +++ b/mpack-instance-manager/src/test/python/instance_manager/test_instance_manager.py @@ -213,6 +213,114 @@ class TestInstanceManager(TestCase): } self.assertEqual(conf_dir_json, expected_json) + def test_get_log_dir_all(self): + create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME.upper()) + create_mpack_with_defaults(module_name=SERVER_MODULE_NAME.upper(), components=None, + components_map={SERVER_COMPONENT_NAME.upper(): ['server1']}) + + log_dir_json = instance_manager.get_log_dir() + + expected_json = { + "mpacks": { + "hdpcore": { + "mpack-instances": { + "Production": { + "name": "Production", + "subgroups": { + "default": { + "modules": { + "hdfs": { + "category": "SERVER", + "name": "hdfs", + "components": { + "hdfs_server": { + "component-instances": { + "server1": { + "log_dir": "/tmp/instance_manager_test/instances/hdpcore/Production/default/hdfs/hdfs_server/server1/log", + "name": "server1" + } + } + } + } + }, + "hdfs-clients": { + "category": "CLIENT", + "components": { + "hdfs_client": { + "component-instances": { + "default": { + "log_dir": "/tmp/instance_manager_test/instances/hdpcore/Production/default/hdfs_client/log", + "name": "default" + } + } + } + }, + "name": "hdfs-clients" + } + } + } + } + } + } + } + } + } + self.assertEqual(log_dir_json, expected_json) + + def test_get_run_dir_all(self): + create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME.upper()) + create_mpack_with_defaults(module_name=SERVER_MODULE_NAME.upper(), components=None, + components_map={SERVER_COMPONENT_NAME.upper(): ['server1']}) + + run_dir_json = instance_manager.get_run_dir() + + expected_json = { + "mpacks": { + "hdpcore": { + "mpack-instances": { + "Production": { + "name": "Production", + "subgroups": { + "default": { + "modules": { + "hdfs": { + "category": "SERVER", + "name": "hdfs", + "components": { + "hdfs_server": { + "component-instances": { + "server1": { + "name": "server1", + "run_dir": "/tmp/instance_manager_test/instances/hdpcore/Production/default/hdfs/hdfs_server/server1/run" + } + } + } + } + }, + "hdfs-clients": { + "category": "CLIENT", + "components": { + "hdfs_client": { + "component-instances": { + "default": { + "name": "default", + "run_dir": "/tmp/instance_manager_test/instances/hdpcore/Production/default/hdfs_client/run" + } + } + } + }, + "name": "hdfs-clients" + } + } + } + } + } + } + } + } + } + self.assertEqual(run_dir_json, expected_json) + def test_list_instances_all(self): create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME.upper()) create_mpack_with_defaults(module_name=SERVER_MODULE_NAME.upper(), components=None, -- To stop receiving notification emails like this one, please contact [email protected].
