http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-kafka.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-kafka.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-kafka.json.j2 new file mode 100644 index 0000000..ecce706 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-kafka.json.j2 @@ -0,0 +1,105 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"kafka_controller", + "rowtype":"service", + "path":"{{kafka_log_dir}}/controller.log" + }, + { + "type":"kafka_request", + "rowtype":"service", + "path":"{{kafka_log_dir}}/kafka-request.log" + }, + { + "type":"kafka_logcleaner", + "rowtype":"service", + "path":"{{kafka_log_dir}}/log-cleaner.log" + }, + { + "type":"kafka_server", + "rowtype":"service", + "path":"{{kafka_log_dir}}/server.log" + }, + { + "type":"kafka_statechange", + "rowtype":"service", + "path":"{{kafka_log_dir}}/state-change.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "kafka_controller", + "kafka_request", + "kafka_logcleaner" + ] + + } + + }, + "log4j_format":"[%d] %p %m (%c)%n", + "multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])", + "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "comment":"Suppose to be same log4j pattern as other kafka processes, but some reason thread is not printed", + "conditions":{ + "fields":{ + "type":[ + "kafka_server", + "kafka_statechange" + ] + + } + + }, + "log4j_format":"[%d] %p %m (%c)%n", + "multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])", + "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + } + + ] + +} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-knox.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-knox.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-knox.json.j2 new file mode 100644 index 0000000..7c8e635 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-knox.json.j2 @@ -0,0 +1,68 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"knox_gateway", + "rowtype":"service", + "path":"{{knox_log_dir}}/gateway.log" + }, + { + "type":"knox_cli", + "rowtype":"service", + "path":"{{knox_log_dir}}/knoxcli.log" + }, + { + "type":"knox_ldap", + "rowtype":"service", + "path":"{{knox_log_dir}}/ldap.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "knox_gateway", + "knox_cli", + "knox_ldap" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2 new file mode 100644 index 0000000..cf338da --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2 @@ -0,0 +1,68 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"logsearch_app", + "rowtype":"service", + "path":"{{logsearch_log_dir}}/logsearch.log" + }, + { + "type":"logsearch_feeder", + "rowtype":"service", + "path":"{{logfeeder_log_dir}}/logfeeder.log" + }, + { + "type":"logsearch_perf", + "rowtype":"service", + "path":"{{logsearch_log_dir}}/logsearch-performance.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "logsearch_app", + "logsearch_feeder", + "logsearch_perf" + ] + + } + + }, + "log4j_format":"%d [%t] %-5p %C{6} (%F:%L) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-oozie.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-oozie.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-oozie.json.j2 new file mode 100644 index 0000000..00cf789 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-oozie.json.j2 @@ -0,0 +1,56 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"oozie_app", + "rowtype":"service", + "path":"{{oozie_log_dir}}/oozie.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "oozie_app" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %5p %c{1}:%L - SERVER[${oozie.instance.id}] %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{DATA:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ranger.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ranger.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ranger.json.j2 new file mode 100644 index 0000000..29a280a --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ranger.json.j2 @@ -0,0 +1,122 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"ranger_admin", + "rowtype":"service", + "path":"{{ranger_admin_log_dir}}/xa_portal.log" + }, + { + "type":"ranger_dbpatch", + "is_enabled":"true", + "path":"{{ranger_admin_log_dir}}/ranger_db_patch.log" + }, + { + "type":"ranger_kms", + "rowtype":"service", + "path":"{{ranger_kms_log_dir}}/kms.log" + }, + { + "type":"ranger_usersync", + "rowtype":"service", + "path":"{{ranger_usersync_log_dir}}/usersync.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ranger_admin", + "ranger_dbpatch" + ] + + } + + }, + "log4j_format":"%d [%t] %-5p %C{6} (%F:%L) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ranger_kms" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p %c{1} - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ranger_usersync" + ] + + } + + }, + "log4j_format":"%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n", + "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})", + "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"dd MMM yyyy HH:mm:ss" + } + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-storm.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-storm.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-storm.json.j2 new file mode 100644 index 0000000..9995689 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-storm.json.j2 @@ -0,0 +1,86 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"storm_drpc", + "rowtype":"service", + "path":"{{storm_log_dir}}/drpc.log" + }, + { + "type":"storm_logviewer", + "rowtype":"service", + "path":"{{storm_log_dir}}/logviewer.log" + }, + { + "type":"storm_nimbus", + "rowtype":"service", + "path":"{{storm_log_dir}}/nimbus.log" + }, + { + "type":"storm_supervisor", + "rowtype":"service", + "path":"{{storm_log_dir}}/supervisor.log" + }, + { + "type":"storm_ui", + "rowtype":"service", + "path":"{{storm_log_dir}}/ui.log" + }, + { + "type":"storm_worker", + "rowtype":"service", + "path":"{{storm_log_dir}}/*worker*.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "storm_drpc", + "storm_logviewer", + "storm_nimbus", + "storm_supervisor", + "storm_ui", + "storm_worker" + ] + + } + + }, + "log4j_format":"", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss.SSS" + } + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2 new file mode 100644 index 0000000..8df7fb6 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2 @@ -0,0 +1,86 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"mapred_historyserver", + "rowtype":"service", + "path":"{{mapred_log_dir_prefix}}/mapred/mapred-mapred-historyserver*.log" + }, + { + "type":"yarn_nodemanager", + "rowtype":"service", + "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-nodemanager-*.log" + }, + { + "type":"yarn_resourcemanager", + "rowtype":"service", + "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-resourcemanager-*.log" + }, + { + "type":"yarn_timelineserver", + "rowtype":"service", + "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-timelineserver-*.log" + }, + { + "type":"yarn_historyserver", + "rowtype":"service", + "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-historyserver-*.log" + }, + { + "type":"yarn_jobsummary", + "rowtype":"service", + "path":"{{yarn_log_dir_prefix}}/yarn/hadoop-mapreduce.jobsummary.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "mapred_historyserver", + "yarn_historyserver", + "yarn_jobsummary", + "yarn_nodemanager", + "yarn_resourcemanager", + "yarn_timelineserver" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zookeeper.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zookeeper.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zookeeper.json.j2 new file mode 100644 index 0000000..b90cb6d --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zookeeper.json.j2 @@ -0,0 +1,56 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"zookeeper", + "rowtype":"service", + "path":"{{zk_log_dir}}/zookeeper*.out" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "zookeeper" + ] + + } + + }, + "log4j_format":"%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}-%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\@%{INT:line_number}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logfeeder.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logfeeder.properties.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logfeeder.properties.j2 new file mode 100644 index 0000000..1edf16a --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logfeeder.properties.j2 @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +logfeeder.checkpoint.folder={{logfeeder_checkpoint_folder}} +metrics.collector.hosts={{metrics_collector_hosts}} +config.files={{logfeeder_config_files}} http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2 new file mode 100644 index 0000000..1e183bd --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2 @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#solr.url=http://solr_host:{{logsearch_solr_port}}/solr +solr.zkhosts={{zookeeper_quorum}}{{logsearch_solr_znode}} + +# Service Logs +solr.core.logs={{logsearch_solr_collection_service_logs}} + +solr.service_logs.split_interval_mins={{service_logs_collection_splits_interval_mins}} +solr.service_logs.shards={{logsearch_numshards}} +solr.service_logs.replication_factor={{logsearch_repfactor}} + +solr.servicelogs.fields={{logsearch_service_logs_fields}} + +# Audit logs +auditlog.solr.zkhosts={{solr_audit_logs_zk_quorum}}{{solr_audit_logs_zk_node}} +auditlog.solr.core.logs={{logsearch_solr_collection_audit_logs}} +auditlog.solr.url={{solr_audit_logs_url}} + +solr.audit_logs.split_interval_mins={{audit_logs_collection_splits_interval_mins}} +solr.audit_logs.shards={{logsearch_numshards}} +solr.audit_logs.replication_factor={{logsearch_repfactor}} + +# History logs +solr.core.history=history +solr.history.config_name=history +solr.history.replication_factor={{logsearch_repfactor}} http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/output.config.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/output.config.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/output.config.json.j2 new file mode 100644 index 0000000..bd88ba7 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/output.config.json.j2 @@ -0,0 +1,97 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "output":[ + { + "is_enabled":"{{solr_service_logs_enable}}", + "comment":"Output to solr for service logs", + "destination":"solr", + "zk_hosts":"{{zookeeper_quorum}}{{logsearch_solr_znode}}", + "collection":"{{logsearch_solr_collection_service_logs}}", + "number_of_shards": "{{logsearch_numshards}}", + "splits_interval_mins": "{{service_logs_collection_splits_interval_mins}}", + "conditions":{ + "fields":{ + "rowtype":[ + "service" + ] + + } + + } + + }, + { + "comment":"Output to solr for audit records", + "is_enabled":"{{solr_audit_logs_enable}}", + "destination":"solr", + "zk_hosts":"{{zookeeper_quorum}}{{logsearch_solr_znode}}", + "collection":"{{logsearch_solr_collection_audit_logs}}", + "number_of_shards": "{{logsearch_numshards}}", + "splits_interval_mins": "{{audit_logs_collection_splits_interval_mins}}", + "conditions":{ + "fields":{ + "rowtype":[ + "audit" + ] + + } + + } + + }, + { + "is_enabled":"{{kafka_service_logs_enable}}", + "destination":"kafka", + "broker_list":"{{kafka_broker_list}}", + "topic":"{{kafka_topic_service_logs}}", + "kafka.security.protocol":"{{kafka_security_protocol}}", + "kafka.sasl.kerberos.service.name":"{{kafka_kerberos_service_name}}", + "conditions":{ + "fields":{ + "rowtype":[ + "service" + ] + + } + + } + + }, + { + "is_enabled":"{{kafka_audit_logs_enable}}", + "destination":"kafka", + "broker_list":"{{kafka_broker_list}}", + "topic":"{{kafka_topic_audit_logs}}", + "kafka.security.protocol":"{{kafka_security_protocol}}", + "kafka.sasl.kerberos.service.name":"{{kafka_kerberos_service_name}}", + "conditions":{ + "fields":{ + "rowtype":[ + "audit" + ] + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/zoo.cfg.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/zoo.cfg.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/zoo.cfg.j2 new file mode 100644 index 0000000..6210f06 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/zoo.cfg.j2 @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The number of milliseconds of each tick +tickTime=2000 +# The number of ticks that the initial +# synchronization phase can take +initLimit=10 +# The number of ticks that can pass between +# sending a request and getting an acknowledgement +syncLimit=5 + +# the directory where the snapshot is stored. +# dataDir=/opt/zookeeper/data +# NOTE: Solr defaults the dataDir to $solrHome/zoo_data + +# the port at which the clients will connect +# clientPort=2181 +# NOTE: Solr sets this based on zkRun / zkHost params \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/quicklinks/quicklinks.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/quicklinks/quicklinks.json b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/quicklinks/quicklinks.json new file mode 100644 index 0000000..16e1f4c --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/quicklinks/quicklinks.json @@ -0,0 +1,28 @@ +{ + "name": "default", + "description": "default quick links configuration", + "configuration": { + "protocol": + { + "type":"http" + }, + "links" : [ + { + "name": "logsearch_ui", + "label": "Logsearch UI", + "url":"%@://%@:%@", + "requires_user_name": "false", + "port":{ + "http_property": "logsearch.ui.port", + "http_default_port": "8888", + "https_property": "logsearch.ui.port", + "https_default_port": "8888", + "regex": "^(\\d+)$", + "site": "logsearch-site" + } + } + ] + + } +} + http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml index 77ff982..b0cff68 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml @@ -174,6 +174,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -210,6 +211,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -322,6 +324,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml index 0ef4674..bf041de 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml @@ -180,6 +180,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -216,6 +217,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -328,6 +330,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json index 9ab413e..d64714a 100755 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json @@ -20,6 +20,9 @@ "KNOX_GATEWAY-START" : ["RANGER_USERSYNC-START", "NAMENODE-START"], "KAFKA_BROKER-START" : ["ZOOKEEPER_SERVER-START", "RANGER_USERSYNC-START", "NAMENODE-START"], "NIMBUS-START" : ["ZOOKEEPER_SERVER-START", "RANGER_USERSYNC-START", "NAMENODE-START"], - "STORM_UI_SERVER-START" : ["NIMBUS-START", "NAMENODE-START"] + "STORM_UI_SERVER-START" : ["NIMBUS-START", "NAMENODE-START"], + "LOGSEARCH_SOLR-START" : ["ZOOKEEPER_SERVER-START"], + "LOGSEARCH_SERVER-START": ["LOGSEARCH_SOLR-START"], + "LOGSEARCH_LOGFEEDER-START": ["LOGSEARCH_SOLR-START", "LOGSEARCH_SERVER-START"] } } http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/stacks/HDP/2.3/services/LOGSEARCH/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/LOGSEARCH/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/LOGSEARCH/metainfo.xml new file mode 100644 index 0000000..df697dc --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/LOGSEARCH/metainfo.xml @@ -0,0 +1,26 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>LOGSEARCH</name> + <extends>common-services/LOGSEARCH/0.5.0</extends> + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml index ed30846..6b74af0 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml @@ -175,6 +175,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -213,6 +214,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -339,6 +341,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml index 4731631..9fb2bba 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml @@ -166,6 +166,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -203,6 +204,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -320,6 +322,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml index b53a090..e3bc7a3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml @@ -173,6 +173,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -211,6 +212,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -337,6 +339,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py new file mode 100644 index 0000000..ab11ba1 --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +from stacks.utils.RMFTestCase import RMFTestCase, Template, InlineTemplate, StaticFile +from resource_management.core.exceptions import ComponentIsNotRunning + +class TestLogFeeder(RMFTestCase): + COMMON_SERVICES_PACKAGE_DIR = "LOGSEARCH/0.5.0/package" + STACK_VERSION = "2.4" + + def configureResourcesCalled(self): + self.assertResourceCalled('Directory', '/var/log/ambari-logsearch-logfeeder', + owner = 'logfeeder', + group = 'logfeeder', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + self.assertResourceCalled('Directory', '/var/run/ambari-logsearch-logfeeder', + owner = 'logfeeder', + group = 'logfeeder', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-logfeeder', + owner = 'logfeeder', + group = 'logfeeder', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + self.assertResourceCalled('Directory', '/etc/ambari-logsearch-logfeeder/conf', + owner = 'logfeeder', + group = 'logfeeder', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + self.assertResourceCalled('Directory', '/etc/ambari-logsearch-logfeeder/conf/checkpoints', + owner = 'logfeeder', + group = 'logfeeder', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + + self.assertResourceCalled('File', '/var/log/ambari-logsearch-logfeeder/logfeeder.out', + owner = 'logfeeder', + group = 'logfeeder', + mode = 0644, + content = '' + ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/logfeeder.properties', + owner = 'logfeeder', + content = Template('logfeeder.properties.j2') + ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/logfeeder-env.sh', + owner = 'logfeeder', + content = InlineTemplate(self.getConfig()['configurations']['logfeeder-env']['content']) + ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/log4j.xml', + owner = 'logfeeder', + content = InlineTemplate(self.getConfig()['configurations']['logfeeder-log4j']['content']) + ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/grok-patterns', + owner = 'logfeeder', + content = Template('grok-patterns.j2'), + encoding = 'utf-8' + ) + + logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'hbase', 'hdfs', 'hive', 'kafka', + 'knox', 'logsearch', 'oozie', 'ranger', 'storm', 'yarn', 'zookeeper'] + + logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag in logfeeder_supported_services] + + for file_name in logfeeder_config_file_names: + self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/' + file_name, + owner = 'logfeeder', + content = Template(file_name + ".j2") + ) + + def test_configure_default(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logfeeder.py", + classname = "LogFeeder", + command = "configure", + config_file = "default.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.configureResourcesCalled() + self.assertNoMoreResources() + + def test_start_default(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logfeeder.py", + classname = "LogFeeder", + command = "start", + config_file = "default.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.configureResourcesCalled() + self.assertResourceCalled('Execute', '/usr/lib/ambari-logsearch-logfeeder/run.sh', + environment={'LOGFEEDER_INCLUDE': '/etc/ambari-logsearch-logfeeder/conf/logfeeder-env.sh'}, + user = 'logfeeder' + ) + + def test_stop_default(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logfeeder.py", + classname = "LogFeeder", + command = "stop", + config_file = "default.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.assertResourceCalled('Execute', 'kill `cat /var/run/ambari-logsearch-logfeeder/logfeeder.pid`', + user = 'logfeeder', + only_if = 'test -f /var/run/ambari-logsearch-logfeeder/logfeeder.pid' + ) + self.assertResourceCalled('File', '/var/run/ambari-logsearch-logfeeder/logfeeder.pid', + action = ['delete'] + ) + +# def test_status_default(self): +# with self.assertRaises(ComponentIsNotRunning) as e: +# self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logfeeder.py", +# classname = "LogFeeder", +# command = "status", +# config_file = "default.json", +# stack_version = self.STACK_VERSION, +# target = RMFTestCase.TARGET_COMMON_SERVICES +# ) http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py new file mode 100644 index 0000000..a60cf94 --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +from stacks.utils.RMFTestCase import RMFTestCase, Template, InlineTemplate, StaticFile +from resource_management.core.exceptions import ComponentIsNotRunning +from mock.mock import MagicMock, patch +from resource_management.libraries.script.config_dictionary import UnknownConfiguration + +class TestLogSearch(RMFTestCase): + COMMON_SERVICES_PACKAGE_DIR = "LOGSEARCH/0.5.0/package" + STACK_VERSION = "2.4" + + def configureResourcesCalled(self): + self.assertResourceCalled('Directory', '/var/log/ambari-logsearch-portal', + owner = 'logsearch', + group = 'logsearch', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + self.assertResourceCalled('Directory', '/var/run/ambari-logsearch-portal', + owner = 'logsearch', + group = 'logsearch', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-portal', + owner = 'logsearch', + group = 'logsearch', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + self.assertResourceCalled('Directory', '/etc/ambari-logsearch-portal/conf', + owner = 'logsearch', + group = 'logsearch', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + + self.assertResourceCalled('File', '/var/log/ambari-logsearch-portal/logsearch.out', + owner = 'logsearch', + group = 'logsearch', + mode = 0644, + content = '' + ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/logsearch.properties', + owner = 'logsearch', + content = Template('logsearch.properties.j2') + ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/log4j.xml', + owner = 'logsearch', + content = InlineTemplate(self.getConfig()['configurations']['logsearch-log4j']['content']) + ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/logsearch-env.sh', + content = InlineTemplate(self.getConfig()['configurations']['logsearch-env']['content']), + mode = 0755, + owner = "logsearch" + ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/solr_configsets/hadoop_logs/conf/solrconfig.xml', + owner = 'logsearch', + content = InlineTemplate(self.getConfig()['configurations']['logsearch-service_logs-solrconfig']['content']) + ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/solr_configsets/audit_logs/conf/solrconfig.xml', + owner = 'logsearch', + content = InlineTemplate(self.getConfig()['configurations']['logsearch-audit_logs-solrconfig']['content']) + ) + + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:None/logsearch -cmd downconfig -confdir /tmp/solr_config_hadoop_logs_0.[0-9]* -confname hadoop_logs$', + only_if = "^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:None/logsearch -cmd get /configs/hadoop_logs$" + ) + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:None/logsearch -cmd upconfig -confdir /etc/ambari-logsearch-portal/conf/solr_configsets/hadoop_logs/conf -confname hadoop_logs$', + not_if = "^test -d /tmp/solr_config_hadoop_logs_0.[0-9]*$" + ) + + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:None/logsearch -cmd downconfig -confdir /tmp/solr_config_history_0.[0-9]* -confname history$', + only_if = "^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:None/logsearch -cmd get /configs/history$" + ) + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:None/logsearch -cmd upconfig -confdir /etc/ambari-logsearch-portal/conf/solr_configsets/history/conf -confname history$', + not_if = "^test -d /tmp/solr_config_history_0.[0-9]*$" + ) + + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:None/logsearch -cmd downconfig -confdir /tmp/solr_config_audit_logs_0.[0-9]* -confname audit_logs$', + only_if = "^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:None/logsearch -cmd get /configs/audit_logs$" + ) + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:None/logsearch -cmd upconfig -confdir /etc/ambari-logsearch-portal/conf/solr_configsets/audit_logs/conf -confname audit_logs$', + not_if = "^test -d /tmp/solr_config_audit_logs_0.[0-9]*$" + ) + self.assertResourceCalled('Execute', ('chmod', '-R', 'ugo+r', '/etc/ambari-logsearch-portal/conf/solr_configsets'), + sudo=True + ) + + def test_configure_default(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logsearch.py", + classname = "LogSearch", + command = "configure", + config_file = "default.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.configureResourcesCalled() + self.assertNoMoreResources() + + def test_start_default(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logsearch.py", + classname = "LogSearch", + command = "start", + config_file = "default.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.configureResourcesCalled() + self.assertResourceCalled('Execute', "/usr/lib/ambari-logsearch-portal/run.sh 61888", + environment = {'LOGSEARCH_INCLUDE': '/etc/ambari-logsearch-portal/conf/logsearch-env.sh'}, + user = "logsearch" + ) + + def test_stop_default(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logsearch.py", + classname = "LogSearch", + command = "stop", + config_file = "default.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.assertResourceCalled('Execute', 'kill `cat /var/run/ambari-logsearch-portal/logsearch.pid` >/dev/null 2>&1', + user = 'logsearch', + only_if = 'test -f /var/run/ambari-logsearch-portal/logsearch.pid' + ) + self.assertResourceCalled('File', '/var/run/ambari-logsearch-portal/logsearch.pid', + action = ['delete'] + ) + +# def test_status_default(self): +# with self.assertRaises(ComponentIsNotRunning) as e: +# self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logsearch.py", +# classname = "LogSearch", +# command = "status", +# config_file = "default.json", +# stack_version = self.STACK_VERSION, +# target = RMFTestCase.TARGET_COMMON_SERVICES +# ) http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_solr.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_solr.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_solr.py new file mode 100644 index 0000000..5a684e6 --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_solr.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +from stacks.utils.RMFTestCase import RMFTestCase, Template, InlineTemplate, StaticFile +from resource_management.core.exceptions import ComponentIsNotRunning +from resource_management.libraries.script.config_dictionary import UnknownConfiguration + +class TestSolr(RMFTestCase): + COMMON_SERVICES_PACKAGE_DIR = "LOGSEARCH/0.5.0/package" + STACK_VERSION = "2.4" + + def configureResourcesCalled(self): + self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-solr', + owner = 'solr', + group = 'solr', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + self.assertResourceCalled('Directory', '/var/log/ambari-logsearch-solr', + owner = 'solr', + group = 'solr', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + self.assertResourceCalled('Directory', '/var/run/ambari-logsearch-solr', + owner = 'solr', + group = 'solr', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + self.assertResourceCalled('Directory', '/etc/ambari-logsearch-solr/conf', + owner = 'solr', + group = 'solr', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + self.assertResourceCalled('Directory', '/opt/logsearch_solr/data', + owner = 'solr', + group = 'solr', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + self.assertResourceCalled('Directory', '/opt/logsearch_solr/data/resources', + owner = 'solr', + group = 'solr', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + + self.assertResourceCalled('File', '/var/log/ambari-logsearch-solr/solr-install.log', + owner = 'solr', + group = 'solr', + mode = 0644, + content = '' + ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-solr/conf/logsearch-solr-env.sh', + owner = 'solr', + mode = 0755, + content = InlineTemplate(self.getConfig()['configurations']['logsearch-solr-env']['content']) + ) + self.assertResourceCalled('File', '/opt/logsearch_solr/data/solr.xml', + owner = 'solr', + content = InlineTemplate(self.getConfig()['configurations']['logsearch-solr-xml']['content']) + ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-solr/conf/log4j.properties', + owner = 'solr', + content = InlineTemplate(self.getConfig()['configurations']['logsearch-solr-log4j']['content']) + ) + self.assertResourceCalled('File', '/opt/logsearch_solr/data/zoo.cfg', + owner = 'solr', + content = Template('zoo.cfg.j2') + ) + self.assertResourceCalled('Execute', 'export JAVA_HOME=/usr/jdk64/jdk1.7.0_45; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org -cmd makepath /logsearch', + not_if = "export JAVA_HOME=/usr/jdk64/jdk1.7.0_45; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org -cmd get /logsearch", + ignore_failures = True, + user = "solr" + ) + + def test_configure_default(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/solr.py", + classname = "Solr", + command = "configure", + config_file = "default.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.configureResourcesCalled() + self.assertNoMoreResources() + + def test_start_default(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/solr.py", + classname = "Solr", + command = "start", + config_file = "default.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.configureResourcesCalled() + self.assertResourceCalled('Execute', "/usr/lib/ambari-logsearch-solr/bin/solr start -cloud -noprompt -s /opt/logsearch_solr/data >> /var/log/ambari-logsearch-solr/solr-install.log 2>&1", + environment = {'SOLR_INCLUDE': '/etc/ambari-logsearch-solr/conf/logsearch-solr-env.sh'}, + user = "solr" + ) + + def test_stop_default(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/solr.py", + classname = "Solr", + command = "stop", + config_file = "default.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.assertResourceCalled('Execute', '/usr/lib/ambari-logsearch-solr/bin/solr stop -all >> /var/log/ambari-logsearch-solr/solr-install.log', + environment = {'SOLR_INCLUDE': '/etc/ambari-logsearch-solr/conf/logsearch-solr-env.sh'}, + user = "solr", + only_if = "test -f /var/run/ambari-logsearch-solr/solr-8886.pid" + ) + self.assertResourceCalled('File', '/var/run/ambari-logsearch-solr/solr-8886.pid', + action = ['delete'] + ) + +# def test_status_default(self): +# with self.assertRaises(ComponentIsNotRunning) as e: +# self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/solr.py", +# classname = "Solr", +# command = "status", +# config_file = "default.json", +# stack_version = self.STACK_VERSION, +# target = RMFTestCase.TARGET_COMMON_SERVICES +# )
