Repository: ambari Updated Branches: refs/heads/branch-2.4 6e5422f62 -> d370a4467
AMBARI-17424. Add missing components to LogSearch Part 2 (Miklos Gergely via oleewere) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ca2e62fe Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ca2e62fe Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ca2e62fe Branch: refs/heads/branch-2.4 Commit: ca2e62fe048a4e35d87043c687027e4ab7b896b2 Parents: 6e5422f Author: Miklos Gergely <[email protected]> Authored: Fri Jun 24 17:18:18 2016 +0200 Committer: oleewere <[email protected]> Committed: Fri Jun 24 23:05:41 2016 +0200 ---------------------------------------------------------------------- .../src/main/resources/HadoopServiceConfig.json | 29 ++++++++++ .../FLUME/1.4.0.2.0/metainfo.xml | 6 +++ .../LOGSEARCH/0.5.0/metainfo.xml | 6 +++ .../LOGSEARCH/0.5.0/package/scripts/params.py | 6 ++- .../templates/input.config-flume.json.j2 | 56 ++++++++++++++++++++ .../templates/input.config-hbase.json.j2 | 29 ++++++++++ .../templates/input.config-logsearch.json.j2 | 30 +++++++++++ .../templates/input.config-zeppelin.json.j2 | 56 ++++++++++++++++++++ .../ZEPPELIN/0.6.0.2.5/metainfo.xml | 6 +++ .../stacks/HDP/2.3/services/HBASE/metainfo.xml | 6 +++ .../stacks/2.4/LOGSEARCH/test_logfeeder.py | 4 +- 11 files changed, 230 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/ca2e62fe/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json b/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json index 73400e3..abe692d 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json @@ -100,6 +100,18 @@ ] }, + "flume":{ + "label":"Flume", + "components":[ + { + "name":"flume_handler" + } + ], + "dependencies":[ + + ] + + }, "hbase":{ "label":"HBase", "components":[ @@ -108,6 +120,9 @@ }, { "name":"hbase_regionserver" + }, + { + "name":"hbase_phoenix_server" } ], @@ -272,6 +287,9 @@ }, { "name":"logsearch_perf" + }, + { + "name":"logsearch_solr" } ], @@ -425,6 +443,17 @@ ] }, + "zeppelin":{ + "label":"Zeppelin", + "components":[ + { + "name":"zeppelin" + } + ], + "dependencies":[ + + ] + }, "zookeeper":{ "label":"ZooKeeper", "components":[ http://git-wip-us.apache.org/repos/asf/ambari/blob/ca2e62fe/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/metainfo.xml index 0396c28..b4f1058 100644 --- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/metainfo.xml @@ -35,6 +35,12 @@ <scriptType>PYTHON</scriptType> <timeout>1200</timeout> </commandScript> + <logs> + <log> + <logId>flume_handler</logId> + <primary>true</primary> + </log> + </logs> </component> </components> http://git-wip-us.apache.org/repos/asf/ambari/blob/ca2e62fe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml index e3a8f98..ffa9e13 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml @@ -84,6 +84,12 @@ <script>scripts/logsearch_solr.py</script> <scriptType>PYTHON</scriptType> </commandScript> + <logs> + <log> + <logId>logsearch_solr</logId> + <primary>true</primary> + </log> + </logs> <dependencies> <dependency> <name>LOGSEARCH/LOGSEARCH_SOLR_CLIENT</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/ca2e62fe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py index d21a97f..4bc1004 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py @@ -247,6 +247,7 @@ metrics_monitor_log_dir = default('/configurations/ams-env/metrics_monitor_log_d atlas_log_dir = default('/configurations/atlas-env/metadata_log_dir', '/var/log/atlas') accumulo_log_dir = default('/configurations/accumulo-env/accumulo_log_dir', '/var/log/accumulo') falcon_log_dir = default('/configurations/falcon-env/falcon_log_dir', '/var/log/falcon') +flume_log_dir = default('/configurations/flume-env/flume_log_dir', '/var/log/flume') hbase_log_dir = default('/configurations/hbase-env/hbase_log_dir', '/var/log/hbase') hdfs_log_dir_prefix = default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop') hive_log_dir = default('/configurations/hive-env/hive_log_dir', '/var/log/hive') @@ -259,6 +260,7 @@ ranger_kms_log_dir = default('/configurations/kms-env/kms_log_dir', '/var/log/ra storm_log_dir = default('/configurations/storm-env/storm_log_dir', '/var/log/storm') yarn_log_dir_prefix = default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop') mapred_log_dir_prefix = default('/configurations/mapred-env/mapred_log_dir_prefix', '/var/log/hadoop') +zeppelin_log_dir = default('/configuration/zeppelin-env/zeppelin_log_dir', '/var/log/zeppelin') zk_log_dir = default('/configurations/zookeeper-env/zk_log_dir', '/var/log/zookeeper') ##################################### @@ -328,8 +330,8 @@ logfeeder_checkpoint_folder = default('/configurations/logfeeder-env/logfeeder.c logfeeder_log_filter_enable = str(default('/configurations/logfeeder-properties/logfeeder.log.filter.enable', True)).lower() logfeeder_solr_config_interval = default('/configurations/logfeeder-properties/logfeeder.solr.config.interval', 5) -logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'hbase', 'hdfs', 'hive', 'kafka', - 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'storm', 'yarn', 'zookeeper'] +logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'kafka', + 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'storm', 'yarn', 'zeppelin', 'zookeeper'] logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag in logfeeder_supported_services] http://git-wip-us.apache.org/repos/asf/ambari/blob/ca2e62fe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-flume.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-flume.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-flume.json.j2 new file mode 100644 index 0000000..d1ea632 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-flume.json.j2 @@ -0,0 +1,56 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"flume_handler", + "rowtype":"service", + "path":"{{flume_log_dir}}/flume.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "flume_handler" + ] + + } + + }, + "log4j_format":"", + "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})", + "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}\\(%{JAVACLASS:class}\\.%{JAVAMETHOD:method}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"dd MMM yyyy HH:mm:ss,SSS" + } + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ca2e62fe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2 index fb47e77..42c5683 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2 @@ -26,6 +26,11 @@ "type":"hbase_regionserver", "rowtype":"service", "path":"{{hbase_log_dir}}/hbase-hbase-regionserver-*.log" + }, + { + "type":"hbase_phoenix_server", + "rowtype":"service", + "path":"{{hbase_log_dir}}/phoenix-hbase-server.log" } ], @@ -55,6 +60,30 @@ } + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "hbase_phoenix_server" + ] + + } + }, + "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + } ] http://git-wip-us.apache.org/repos/asf/ambari/blob/ca2e62fe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2 index ba151b9..31c37bf 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2 @@ -31,6 +31,11 @@ "type":"logsearch_perf", "rowtype":"service", "path":"{{logsearch_log_dir}}/logsearch-performance.json" + }, + { + "type":"logsearch_solr", + "rowtype":"service", + "path":"{{logsearch_solr_log_dir}}/solr.log" } ], @@ -49,6 +54,31 @@ } + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "logsearch_solr" + ] + + } + + }, + "log4j_format":"", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + } ] http://git-wip-us.apache.org/repos/asf/ambari/blob/ca2e62fe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zeppelin.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zeppelin.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zeppelin.json.j2 new file mode 100644 index 0000000..09cd8b3 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zeppelin.json.j2 @@ -0,0 +1,56 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"zeppelin", + "rowtype":"service", + "path":"{{zeppelin_log_dir}}/zeppelin-zeppelin-*.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "zeppelin" + ] + + } + + }, + "log4j_format":"", + "multiline_pattern":"^(%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{TIMESTAMP_ISO8601:logtime}\\])", + "message_pattern":"(?m)^%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}\\(\\{{"{"}}%{DATA:thread_name}\\{{"}"}}%{SPACE}%{JAVAFILE:file}\\[%{JAVAMETHOD:method}\\]:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ca2e62fe/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml index c40a593..4f19b43 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml @@ -53,6 +53,12 @@ limitations under the License. </auto-deploy> </dependency> </dependencies> + <logs> + <log> + <logId>zeppelin</logId> + <primary>true</primary> + </log> + </logs> </component> </components> http://git-wip-us.apache.org/repos/asf/ambari/blob/ca2e62fe/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/metainfo.xml index 0614cf7..d7401d8 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/metainfo.xml @@ -33,6 +33,12 @@ <script>scripts/phoenix_queryserver.py</script> <scriptType>PYTHON</scriptType> </commandScript> + <logs> + <log> + <logId>hbase_phoenix_server</logId> + <primary>true</primary> + </log> + </logs> </component> </components> http://git-wip-us.apache.org/repos/asf/ambari/blob/ca2e62fe/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py index 0613770..26e8934 100644 --- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py +++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py @@ -98,8 +98,8 @@ class TestLogFeeder(RMFTestCase): encoding='utf-8' ) - logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'hbase', 'hdfs', 'hive', 'kafka', - 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'storm', 'yarn', 'zookeeper'] + logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'kafka', + 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'storm', 'yarn', 'zeppelin', 'zookeeper'] logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag in logfeeder_supported_services]
