Repository: ambari Updated Branches: refs/heads/branch-2.4 a1a98ac8d -> cb39b96c8 refs/heads/trunk 6e44c8d98 -> 4408d779b
AMBARI-16703. Takeover config merge should handle AMS hbase configs (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4408d779 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4408d779 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4408d779 Branch: refs/heads/trunk Commit: 4408d779b8aa9c56eb8d63208003ba1d3f2281ac Parents: 6e44c8d Author: Andrew Onishuk <[email protected]> Authored: Thu May 19 15:53:56 2016 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Thu May 19 15:53:56 2016 +0300 ---------------------------------------------------------------------- .../resources/scripts/takeover_config_merge.py | 33 ++++++++------------ .../scripts/takeover_files_mapping.json | 23 ++++++++++++++ 2 files changed, 36 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4408d779/ambari-server/src/main/resources/scripts/takeover_config_merge.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/scripts/takeover_config_merge.py b/ambari-server/src/main/resources/scripts/takeover_config_merge.py index f940f27..075f99f 100644 --- a/ambari-server/src/main/resources/scripts/takeover_config_merge.py +++ b/ambari-server/src/main/resources/scripts/takeover_config_merge.py @@ -33,7 +33,7 @@ import ConfigParser logger = logging.getLogger('AmbariTakeoverConfigMerge') -LOG4J_HELP_TEXT = """ +CONFIG_MAPPING_HELP_TEXT = """ JSON file should content map with {regex_path : <service>-log4j} Example: {".+/hadoop/.+/log4j.properties" : "hdfs-log4j", @@ -158,13 +158,6 @@ class ConfigMerge: SUPPORTED_EXTENSIONS = ['.xml', '.yaml', '.properties', '.sh'] SUPPORTED_FILENAME_ENDINGS = {".sh" : "-env"} UNKNOWN_FILES_MAPPING_FILE = None - SERVICE_TO_AMBARI_CONFIG_NAME = { - "storm.yaml": "storm-site", - "runtime.properties" : "falcon-runtime", - "startup.properties" : "falcon-startup", - "flume-conf.properties" : "flume-conf", - "pig.properties" : "pig-properties" - } CONFIGS_WITH_CONTENT = ['pig-properties', '-log4j'] @@ -188,11 +181,8 @@ class ConfigMerge: logger.warn("File {0} is not configurable by Ambari. Skipping...".format(file_path)) continue config_name = None - if file in ConfigMerge.SERVICE_TO_AMBARI_CONFIG_NAME: - config_name = ConfigMerge.SERVICE_TO_AMBARI_CONFIG_NAME[file] - #hack for log4j.properties files - elif ConfigMerge.UNKNOWN_FILES_MAPPING_FILE: + if ConfigMerge.UNKNOWN_FILES_MAPPING_FILE: for path_regex, name in ConfigMerge.CONTENT_UNKNOWN_FILES_MAPPING_FILE.iteritems(): match = re.match(path_regex, os.path.relpath(file_path, ConfigMerge.INPUT_DIR)) if match: @@ -364,8 +354,8 @@ def main(): metavar="FILE", help="Output directory. [default: /tmp]") parser.add_option("-i", "--inputdir", dest="inputDir", help="Input directory.") - parser.add_option("-u", '--unknown-files-mapping-file',dest="unknown_files_mapping_file", default=None, - metavar="FILE", help=LOG4J_HELP_TEXT) + parser.add_option("-u", '--unknown-files-mapping-file',dest="unknown_files_mapping_file", + metavar="FILE", help=CONFIG_MAPPING_HELP_TEXT, default="takeover_files_mapping.json") (options, args) = parser.parse_args() @@ -378,12 +368,6 @@ def main(): ConfigMerge.INPUT_DIR = options.inputDir ConfigMerge.OUTPUT_DIR = options.outputDir - #hack for logf4.properties files - if options.unknown_files_mapping_file: - ConfigMerge.UNKNOWN_FILES_MAPPING_FILE = options.unknown_files_mapping_file - with open(options.unknown_files_mapping_file) as f: - ConfigMerge.CONTENT_UNKNOWN_FILES_MAPPING_FILE = json.load(f) - if not os.path.exists(ConfigMerge.OUTPUT_DIR): os.makedirs(ConfigMerge.OUTPUT_DIR) @@ -398,6 +382,15 @@ def main(): stdout_handler.setFormatter(formatter) logger.addHandler(stdout_handler) + #unknown file mapping + if options.unknown_files_mapping_file and os.path.exists(options.unknown_files_mapping_file): + ConfigMerge.UNKNOWN_FILES_MAPPING_FILE = options.unknown_files_mapping_file + with open(options.unknown_files_mapping_file) as f: + ConfigMerge.CONTENT_UNKNOWN_FILES_MAPPING_FILE = json.load(f) + else: + logger.warning("Config mapping file was not found at {0}. " + "Please provide it at the given path or provide a different path to it using -u option.".format(options.unknown_files_mapping_file)) + filePaths = ConfigMerge.get_all_supported_files_grouped_by_name() logger.info("Writing logs into '{0}' file".format(logegr_file_name)) logger.debug("Following configuration files found:\n{0}".format(filePaths.items())) http://git-wip-us.apache.org/repos/asf/ambari/blob/4408d779/ambari-server/src/main/resources/scripts/takeover_files_mapping.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/scripts/takeover_files_mapping.json b/ambari-server/src/main/resources/scripts/takeover_files_mapping.json new file mode 100644 index 0000000..69b4085 --- /dev/null +++ b/ambari-server/src/main/resources/scripts/takeover_files_mapping.json @@ -0,0 +1,23 @@ +{ + ".+/ams-hbase/.+/hbase-site.xml" : "ams-hbase-site", + ".+/ams-hbase/.+/log4j.properties" : "ams-hbase-log4j", + ".+/ambari-metrics-collector/.+/log4j.properties" : "ams-log4j", + ".+/etc/accumulo/.+/log4j.properties" : "accumulo-log4j", + ".+/etc/atlas/.+/log4j.properties" : "atlas-log4j", + ".+/etc/flume/.+/log4j.properties" : "flume-log4j", + ".+/flume-conf.properties" : "flume-conf", + ".+/etc/zookeeper/.+/log4j.properties" : "zookeeper-log4j", + ".+/etc/hbase/.+/log4j.properties" : "hbase-log4j", + ".+/etc/ams-hbase/.+/log4j.properties" : "ams-hbase-log4j", + ".+/etc/pig/.+/log4j.properties" : "pig-log4j", + ".+/pig.properties" : "pig-properties", + ".+/etc/hadoop/.+/log4j.properties" : "hdfs-log4j", + ".+/etc/hive/.+/log4j.properties" : "hive-log4j", + ".+/etc/kafka/.+/log4j.properties" : "kafka-log4j", + ".+/etc/oozie/.+/log4j.properties" : "oozie-log4j", + ".+/etc/slider/.+/log4j.properties" : "slider-log4j", + ".+/etc/ambari-metrics-collector/.+/log4j.properties" : "ams-log4j", + ".+/storm.yaml" : "storm-site", + ".+/runtime.properties" : "falcon-runtime", + ".+/startup.properties" : "falcon-startup" +} \ No newline at end of file
