Repository: ambari Updated Branches: refs/heads/trunk 84a2595eb -> e1069651d
AMBARI-16158. takeover_config_merge.py should process known template config files (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e1069651 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e1069651 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e1069651 Branch: refs/heads/trunk Commit: e1069651db0f503f15c8b31ec50f1932d65ab7be Parents: 84a2595 Author: Andrew Onishuk <[email protected]> Authored: Thu Apr 28 18:56:31 2016 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Thu Apr 28 18:56:31 2016 +0300 ---------------------------------------------------------------------- .../resources/scripts/takeover_config_merge.py | 160 +++++++++++++++---- 1 file changed, 128 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/e1069651/ambari-server/src/main/resources/scripts/takeover_config_merge.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/scripts/takeover_config_merge.py b/ambari-server/src/main/resources/scripts/takeover_config_merge.py index ed1da07..8f11db4 100644 --- a/ambari-server/src/main/resources/scripts/takeover_config_merge.py +++ b/ambari-server/src/main/resources/scripts/takeover_config_merge.py @@ -23,43 +23,67 @@ import os import logging import tempfile import json +import re import base64 import time import xml import xml.etree.ElementTree as ET +import yaml +import StringIO +import ConfigParser logger = logging.getLogger('AmbariTakeoverConfigMerge') +LOG4J_HELP_TEXT = """ +JSON file should content map with {regex_path : <service>-log4j} +Example: +{".+/hadoop/.+/log4j.properties" : "hdfs-log4j", +".+/etc/zookeeper/conf/log4j.properties" : "zookeeper-log4j" +"c6401.ambari.apache.org/etc/hive/conf/log4j.properties" : "hive-log4j"} +""" -class ConfigMerge: - INPUT_DIR = '/etc/hadoop' - OUTPUT_DIR = '/tmp' - OUT_FILENAME = 'ambari_takeover_config_merge.out' - JSON_FILENAME = 'ambari_takeover_config_merge.json' - SUPPORTED_EXTENSIONS = ['.xml'] - SUPPORTED_FILENAME_ENDINGS = ['-site'] +class Parser: + pass - config_files_map = {} +class YamlParser(Parser): # Used Yaml parser to read data into a map + def read_data_to_map(self, path): + configurations = {} + with open(path, 'r') as file: + try: + for name, value in yaml.load(file).iteritems(): + if name != None: + configurations[name] = str(value) + except yaml.YAMLError: + logger.exception("Yaml parser error: ") + return None + return configurations - def __init__(self, config_files_map): - self.config_files_map = config_files_map - @staticmethod - def get_all_supported_files_grouped_by_name(extensions=SUPPORTED_EXTENSIONS): - filePaths = {} - for dirName, subdirList, fileList in os.walk(ConfigMerge.INPUT_DIR, followlinks=True): - for file in fileList: - root, ext = os.path.splitext(file) - for filename_ending in ConfigMerge.SUPPORTED_FILENAME_ENDINGS: - if root.endswith(filename_ending) and ext in extensions: - if not file in filePaths: - filePaths[file] = [] - filePaths[file].append(os.path.join(dirName, file)) +class PropertiesParser(Parser): # Used ConfigParser parser to read data into a map + def read_data_to_map(self, path): + configurations = {} + try : + #Adding dummy section to properties file content for use ConfigParser + properties_file_content = StringIO.StringIO() + properties_file_content.write('[dummysection]\n') + properties_file_content.write(open(path).read()) + properties_file_content.seek(0, os.SEEK_SET) + + cp = ConfigParser.ConfigParser() + cp.readfp(properties_file_content) + + for section in cp._sections: + for name, value in cp._sections[section].iteritems(): + if name != None: + configurations[name] = value + del configurations['__name__'] + except: + logger.exception("ConfigParser error: ") + return configurations - return filePaths - # Used DOM parser to read data into a map - def read_xml_data_to_map(self, path): +class XmlParser(Parser): # Used DOM parser to read data into a map + def read_data_to_map(self, path): configurations = {} tree = ET.parse(path) root = tree.getroot() @@ -83,6 +107,65 @@ class ConfigMerge: logger.debug("Following configurations found in {0}:\n{1}".format(path, configurations)) return configurations +class ConfigMerge: + + CONTENT_UNKNOWN_FILES_MAPPING_FILE = {} + INPUT_DIR = '/etc/hadoop' + OUTPUT_DIR = '/tmp' + OUT_FILENAME = 'ambari_takeover_config_merge.out' + JSON_FILENAME = 'ambari_takeover_config_merge.json' + PARSER_BY_EXTENSIONS = {'.xml' : XmlParser(), '.yaml' : YamlParser(), '.properties' : PropertiesParser()} + SUPPORTED_EXTENSIONS = ['.xml', '.yaml', '.properties'] + UNKNOWN_FILES_MAPPING_FILE = None + SERVICE_TO_AMBARI_CONFIG_NAME = { + "storm.yaml": "storm-site" + } + + NOT_MAPPED_FILES = ['log4j.properties'] + + + config_files_map = {} + + def __init__(self, config_files_map): + self.config_files_map = config_files_map + + @staticmethod + def get_all_supported_files_grouped_by_name(extensions=SUPPORTED_EXTENSIONS): + filePaths = {} + for dirName, subdirList, fileList in os.walk(ConfigMerge.INPUT_DIR, followlinks=True): + for file in fileList: + root, ext = os.path.splitext(file) + if ext in extensions: + file_path = os.path.join(dirName, file) + + config_name = None + if file in ConfigMerge.SERVICE_TO_AMBARI_CONFIG_NAME: + config_name = ConfigMerge.SERVICE_TO_AMBARI_CONFIG_NAME[file] + + #hack for log4j.properties files + elif ConfigMerge.UNKNOWN_FILES_MAPPING_FILE: + for path_regex, name in ConfigMerge.CONTENT_UNKNOWN_FILES_MAPPING_FILE.iteritems(): + match = re.match(path_regex, os.path.relpath(file_path, ConfigMerge.INPUT_DIR)) + if match: + config_name = name + break + + if not config_name: + if file in ConfigMerge.NOT_MAPPED_FILES: + if ConfigMerge.UNKNOWN_FILES_MAPPING_FILE: + logger.error("File {0} doesn't match any regex from {1}".format(file_path, ConfigMerge.UNKNOWN_FILES_MAPPING_FILE)) + else: + logger.error("Cannot map {0} to Ambari config type. Please use -u option to specify config mapping for this file. \n" + "For more information use --help option for script".format(file_path)) + continue + else: + config_name = file + + if not config_name in filePaths: + filePaths[config_name] = [] + filePaths[config_name].append((file_path, ConfigMerge.PARSER_BY_EXTENSIONS[ext])) + return filePaths + @staticmethod def merge_configurations(filepath_to_configurations): configuration_information_dict = {} @@ -131,12 +214,15 @@ class ConfigMerge: def perform_merge(self): result_configurations = {} has_conflicts = False - for filename, paths in self.config_files_map.iteritems(): + for filename, paths_and_parsers in self.config_files_map.iteritems(): filepath_to_configurations = {} configuration_type = os.path.splitext(filename)[0] - for path in paths: - logger.debug("Read xml data from {0}".format(path)) - filepath_to_configurations[path] = self.read_xml_data_to_map(path) + for path_and_parser in paths_and_parsers: + path, parser = path_and_parser + logger.debug("Read data from {0}".format(path)) + parsed_configurations_from_path = parser.read_data_to_map(path) + if parsed_configurations_from_path != None: + filepath_to_configurations[path] = parsed_configurations_from_path merged_configurations, property_name_to_value_to_filepaths = ConfigMerge.merge_configurations( filepath_to_configurations) @@ -164,7 +250,6 @@ class ConfigMerge: logger.info("Script successfully finished") return 0 - def main(): tempDir = tempfile.gettempdir() outputDir = os.path.join(tempDir) @@ -177,9 +262,8 @@ def main(): 'from a target directory and produces an out file ' 'with problems found that need to be addressed and ' 'the json file which can be used to create the ' - 'blueprint.\n\nThis script only works with *-site.xml ' - 'files for now.' - ) + 'blueprint.\n\nThis script only works with *.xml *.yaml ' + 'and *.properties extensions of files.') parser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False, help="output verbosity.") @@ -187,6 +271,9 @@ def main(): metavar="FILE", help="Output directory. [default: /tmp]") parser.add_option("-i", "--inputdir", dest="inputDir", help="Input directory.") + parser.add_option("-u", '--unknown-files-mapping-file',dest="unknown_files_mapping_file", default=None, + metavar="FILE", help=LOG4J_HELP_TEXT) + (options, args) = parser.parse_args() # set verbose @@ -198,6 +285,15 @@ def main(): ConfigMerge.INPUT_DIR = options.inputDir ConfigMerge.OUTPUT_DIR = options.outputDir + #hack for logf4.properties files + if options.unknown_files_mapping_file: + ConfigMerge.UNKNOWN_FILES_MAPPING_FILE = options.unknown_files_mapping_file + with open(options.unknown_files_mapping_file) as f: + ConfigMerge.CONTENT_UNKNOWN_FILES_MAPPING_FILE = json.load(f) + + if not os.path.exists(ConfigMerge.OUTPUT_DIR): + os.makedirs(ConfigMerge.OUTPUT_DIR) + logegr_file_name = os.path.join(ConfigMerge.OUTPUT_DIR, "takeover_config_merge.log") file_handler = logging.FileHandler(logegr_file_name, mode="w")
