Repository: ambari Updated Branches: refs/heads/branch-2.4 08baa93a8 -> be401e9b0 refs/heads/trunk 347fd2fac -> 291577035
AMBARI-16703. Takeover config merge should handle AMS hbase configs (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/29157703 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/29157703 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/29157703 Branch: refs/heads/trunk Commit: 291577035c076435376c224c8d987ad6bd8b702e Parents: 347fd2f Author: Andrew Onishuk <[email protected]> Authored: Thu May 19 13:24:01 2016 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Thu May 19 13:24:01 2016 +0300 ---------------------------------------------------------------------- .../resources/scripts/takeover_config_merge.py | 33 ++++++++------------ .../scripts/takeover_files_mapping.json | 23 ++++++++++++++ nohup.out | 4 +++ 3 files changed, 40 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/29157703/ambari-server/src/main/resources/scripts/takeover_config_merge.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/scripts/takeover_config_merge.py b/ambari-server/src/main/resources/scripts/takeover_config_merge.py index f940f27..075f99f 100644 --- a/ambari-server/src/main/resources/scripts/takeover_config_merge.py +++ b/ambari-server/src/main/resources/scripts/takeover_config_merge.py @@ -33,7 +33,7 @@ import ConfigParser logger = logging.getLogger('AmbariTakeoverConfigMerge') -LOG4J_HELP_TEXT = """ +CONFIG_MAPPING_HELP_TEXT = """ JSON file should content map with {regex_path : <service>-log4j} Example: {".+/hadoop/.+/log4j.properties" : "hdfs-log4j", @@ -158,13 +158,6 @@ class ConfigMerge: SUPPORTED_EXTENSIONS = ['.xml', '.yaml', '.properties', '.sh'] SUPPORTED_FILENAME_ENDINGS = {".sh" : "-env"} UNKNOWN_FILES_MAPPING_FILE = None - SERVICE_TO_AMBARI_CONFIG_NAME = { - "storm.yaml": "storm-site", - "runtime.properties" : "falcon-runtime", - "startup.properties" : "falcon-startup", - "flume-conf.properties" : "flume-conf", - "pig.properties" : "pig-properties" - } CONFIGS_WITH_CONTENT = ['pig-properties', '-log4j'] @@ -188,11 +181,8 @@ class ConfigMerge: logger.warn("File {0} is not configurable by Ambari. Skipping...".format(file_path)) continue config_name = None - if file in ConfigMerge.SERVICE_TO_AMBARI_CONFIG_NAME: - config_name = ConfigMerge.SERVICE_TO_AMBARI_CONFIG_NAME[file] - #hack for log4j.properties files - elif ConfigMerge.UNKNOWN_FILES_MAPPING_FILE: + if ConfigMerge.UNKNOWN_FILES_MAPPING_FILE: for path_regex, name in ConfigMerge.CONTENT_UNKNOWN_FILES_MAPPING_FILE.iteritems(): match = re.match(path_regex, os.path.relpath(file_path, ConfigMerge.INPUT_DIR)) if match: @@ -364,8 +354,8 @@ def main(): metavar="FILE", help="Output directory. [default: /tmp]") parser.add_option("-i", "--inputdir", dest="inputDir", help="Input directory.") - parser.add_option("-u", '--unknown-files-mapping-file',dest="unknown_files_mapping_file", default=None, - metavar="FILE", help=LOG4J_HELP_TEXT) + parser.add_option("-u", '--unknown-files-mapping-file',dest="unknown_files_mapping_file", + metavar="FILE", help=CONFIG_MAPPING_HELP_TEXT, default="takeover_files_mapping.json") (options, args) = parser.parse_args() @@ -378,12 +368,6 @@ def main(): ConfigMerge.INPUT_DIR = options.inputDir ConfigMerge.OUTPUT_DIR = options.outputDir - #hack for logf4.properties files - if options.unknown_files_mapping_file: - ConfigMerge.UNKNOWN_FILES_MAPPING_FILE = options.unknown_files_mapping_file - with open(options.unknown_files_mapping_file) as f: - ConfigMerge.CONTENT_UNKNOWN_FILES_MAPPING_FILE = json.load(f) - if not os.path.exists(ConfigMerge.OUTPUT_DIR): os.makedirs(ConfigMerge.OUTPUT_DIR) @@ -398,6 +382,15 @@ def main(): stdout_handler.setFormatter(formatter) logger.addHandler(stdout_handler) + #unknown file mapping + if options.unknown_files_mapping_file and os.path.exists(options.unknown_files_mapping_file): + ConfigMerge.UNKNOWN_FILES_MAPPING_FILE = options.unknown_files_mapping_file + with open(options.unknown_files_mapping_file) as f: + ConfigMerge.CONTENT_UNKNOWN_FILES_MAPPING_FILE = json.load(f) + else: + logger.warning("Config mapping file was not found at {0}. " + "Please provide it at the given path or provide a different path to it using -u option.".format(options.unknown_files_mapping_file)) + filePaths = ConfigMerge.get_all_supported_files_grouped_by_name() logger.info("Writing logs into '{0}' file".format(logegr_file_name)) logger.debug("Following configuration files found:\n{0}".format(filePaths.items())) http://git-wip-us.apache.org/repos/asf/ambari/blob/29157703/ambari-server/src/main/resources/scripts/takeover_files_mapping.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/scripts/takeover_files_mapping.json b/ambari-server/src/main/resources/scripts/takeover_files_mapping.json new file mode 100644 index 0000000..69b4085 --- /dev/null +++ b/ambari-server/src/main/resources/scripts/takeover_files_mapping.json @@ -0,0 +1,23 @@ +{ + ".+/ams-hbase/.+/hbase-site.xml" : "ams-hbase-site", + ".+/ams-hbase/.+/log4j.properties" : "ams-hbase-log4j", + ".+/ambari-metrics-collector/.+/log4j.properties" : "ams-log4j", + ".+/etc/accumulo/.+/log4j.properties" : "accumulo-log4j", + ".+/etc/atlas/.+/log4j.properties" : "atlas-log4j", + ".+/etc/flume/.+/log4j.properties" : "flume-log4j", + ".+/flume-conf.properties" : "flume-conf", + ".+/etc/zookeeper/.+/log4j.properties" : "zookeeper-log4j", + ".+/etc/hbase/.+/log4j.properties" : "hbase-log4j", + ".+/etc/ams-hbase/.+/log4j.properties" : "ams-hbase-log4j", + ".+/etc/pig/.+/log4j.properties" : "pig-log4j", + ".+/pig.properties" : "pig-properties", + ".+/etc/hadoop/.+/log4j.properties" : "hdfs-log4j", + ".+/etc/hive/.+/log4j.properties" : "hive-log4j", + ".+/etc/kafka/.+/log4j.properties" : "kafka-log4j", + ".+/etc/oozie/.+/log4j.properties" : "oozie-log4j", + ".+/etc/slider/.+/log4j.properties" : "slider-log4j", + ".+/etc/ambari-metrics-collector/.+/log4j.properties" : "ams-log4j", + ".+/storm.yaml" : "storm-site", + ".+/runtime.properties" : "falcon-runtime", + ".+/startup.properties" : "falcon-startup" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/29157703/nohup.out ---------------------------------------------------------------------- diff --git a/nohup.out b/nohup.out new file mode 100644 index 0000000..363da0e --- /dev/null +++ b/nohup.out @@ -0,0 +1,4 @@ +[17613:17649:0518/184644:ERROR:nss_util.cc(839)] After loading Root Certs, loaded==false: NSS error code: -8018 +Created new window in existing browser session. +[17755:17792:0518/184909:ERROR:nss_util.cc(839)] After loading Root Certs, loaded==false: NSS error code: -8018 +Created new window in existing browser session.
