http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2 new file mode 100644 index 0000000..730ef99 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2 @@ -0,0 +1,246 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"hdfs_datanode", + "rowtype":"service", + "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-datanode-*.log" + }, + { + "type":"hdfs_namenode", + "rowtype":"service", + "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-namenode-*.log" + }, + { + "type":"hdfs_journalnode", + "rowtype":"service", + "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-journalnode-*.log" + }, + { + "type":"hdfs_secondarynamenode", + "rowtype":"service", + "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-secondarynamenode-*.log" + }, + { + "type":"hdfs_zkfc", + "rowtype":"service", + "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-zkfc-*.log" + }, + { + "type":"hdfs_audit", + "rowtype":"audit", + "is_enabled":"true", + "add_fields":{ + "logType":"HDFSAudit", + "enforcer":"hadoop-acl", + "repoType":"1", + "repo":"hdfs" + }, + "path":"{{hdfs_log_dir_prefix}}/hdfs/hdfs-audit.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "hdfs_datanode", + "hdfs_journalnode", + "hdfs_secondarynamenode", + "hdfs_namenode", + "hdfs_zkfc" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "hdfs_audit" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:evtTime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:evtTime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "evtTime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"keyvalue", + "sort_order":1, + "conditions":{ + "fields":{ + "type":[ + "hdfs_audit" + ] + + } + + }, + "source_field":"log_message", + "value_split":"=", + "field_split":"\t", + "post_map_values":{ + "src":{ + "map_fieldname":{ + "new_fieldname":"resource" + } + + }, + "ip":{ + "map_fieldname":{ + "new_fieldname":"cliIP" + } + + }, + "allowed":[ + { + "map_fieldvalue":{ + "pre_value":"true", + "post_value":"1" + } + + }, + { + "map_fieldvalue":{ + "pre_value":"false", + "post_value":"0" + } + + }, + { + "map_fieldname":{ + "new_fieldname":"result" + } + + } + + ], + "cmd":{ + "map_fieldname":{ + "new_fieldname":"action" + } + + }, + "proto":{ + "map_fieldname":{ + "new_fieldname":"cliType" + } + + }, + "callerContext":{ + "map_fieldname":{ + "new_fieldname":"req_caller_id" + } + + } + + } + + }, + { + "filter":"grok", + "sort_order":2, + "source_field":"ugi", + "remove_source_field":"false", + "conditions":{ + "fields":{ + "type":[ + "hdfs_audit" + ] + + } + + }, + "message_pattern":"%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}", + "post_map_values":{ + "user":{ + "map_fieldname":{ + "new_fieldname":"reqUser" + } + + }, + "x_user":{ + "map_fieldname":{ + "new_fieldname":"reqUser" + } + + }, + "p_user":{ + "map_fieldname":{ + "new_fieldname":"reqUser" + } + + }, + "k_user":{ + "map_fieldname":{ + "new_fieldname":"proxyUsers" + } + + }, + "p_authType":{ + "map_fieldname":{ + "new_fieldname":"authType" + } + + }, + "k_authType":{ + "map_fieldname":{ + "new_fieldname":"proxyAuthType" + } + + } + + } + + } + + ] + +} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2 new file mode 100644 index 0000000..131ae86 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2 @@ -0,0 +1,62 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"hive_hiveserver2", + "rowtype":"service", + "path":"{{hive_log_dir}}/hiveserver2.log" + }, + { + "type":"hive_metastore", + "rowtype":"service", + "path":"{{hive_log_dir}}/hivemetastore.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "hive_hiveserver2", + "hive_metastore" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]:%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-kafka.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-kafka.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-kafka.json.j2 new file mode 100644 index 0000000..73e501d --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-kafka.json.j2 @@ -0,0 +1,105 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"kafka_controller", + "rowtype":"service", + "path":"{{kafka_log_dir}}/controller.log" + }, + { + "type":"kafka_request", + "rowtype":"service", + "path":"{{kafka_log_dir}}/kafka-request.log" + }, + { + "type":"kafka_logcleaner", + "rowtype":"service", + "path":"{{kafka_log_dir}}/log-cleaner.log" + }, + { + "type":"kafka_server", + "rowtype":"service", + "path":"{{kafka_log_dir}}/server.log" + }, + { + "type":"kafka_statechange", + "rowtype":"service", + "path":"{{kafka_log_dir}}/state-change.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "kafka_controller", + "kafka_request", + "kafka_logcleaner" + ] + + } + + }, + "log4j_format":"[%d] %p %m (%c)%n", + "multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])", + "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "comment":"Suppose to be same log4j pattern as other kafka processes, but some reason thread is not printed", + "conditions":{ + "fields":{ + "type":[ + "kafka_server", + "kafka_statechange" + ] + + } + + }, + "log4j_format":"[%d] %p %m (%c)%n", + "multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])", + "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-knox.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-knox.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-knox.json.j2 new file mode 100644 index 0000000..50c2c6d --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-knox.json.j2 @@ -0,0 +1,68 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"knox_gateway", + "rowtype":"service", + "path":"{{knox_log_dir}}/gateway.log" + }, + { + "type":"knox_cli", + "rowtype":"service", + "path":"{{knox_log_dir}}/knoxcli.log" + }, + { + "type":"knox_ldap", + "rowtype":"service", + "path":"{{knox_log_dir}}/ldap.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "knox_gateway", + "knox_cli", + "knox_ldap" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2 new file mode 100644 index 0000000..d9ef66d --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2 @@ -0,0 +1,68 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"logsearch_app", + "rowtype":"service", + "path":"{{logsearch_log_dir}}/logsearch.log" + }, + { + "type":"logsearch_feeder", + "rowtype":"service", + "path":"{{logfeeder_log_dir}}/logfeeder.log" + }, + { + "type":"logsearch_perf", + "rowtype":"service", + "path":"{{logsearch_log_dir}}/logsearch-performance.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "logsearch_app", + "logsearch_feeder", + "logsearch_perf" + ] + + } + + }, + "log4j_format":"%d [%t] %-5p %C{6} (%F:%L) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-oozie.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-oozie.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-oozie.json.j2 new file mode 100644 index 0000000..fc125ec --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-oozie.json.j2 @@ -0,0 +1,56 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"oozie_app", + "rowtype":"service", + "path":"{{oozie_log_dir}}/oozie.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "oozie_app" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %5p %c{1}:%L - SERVER[${oozie.instance.id}] %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{DATA:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ranger.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ranger.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ranger.json.j2 new file mode 100644 index 0000000..8ec0153 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ranger.json.j2 @@ -0,0 +1,122 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"ranger_admin", + "rowtype":"service", + "path":"{{ranger_admin_log_dir}}/xa_portal.log" + }, + { + "type":"ranger_dbpatch", + "is_enabled":"true", + "path":"{{ranger_admin_log_dir}}/ranger_db_patch.log" + }, + { + "type":"ranger_kms", + "rowtype":"service", + "path":"{{ranger_kms_log_dir}}/kms.log" + }, + { + "type":"ranger_usersync", + "rowtype":"service", + "path":"{{ranger_usersync_log_dir}}/usersync.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ranger_admin", + "ranger_dbpatch" + ] + + } + + }, + "log4j_format":"%d [%t] %-5p %C{6} (%F:%L) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ranger_kms" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p %c{1} - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ranger_usersync" + ] + + } + + }, + "log4j_format":"%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n", + "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})", + "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"dd MMM yyyy HH:mm:ss" + } + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-storm.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-storm.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-storm.json.j2 new file mode 100644 index 0000000..e8e95c3 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-storm.json.j2 @@ -0,0 +1,86 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"storm_drpc", + "rowtype":"service", + "path":"{{storm_log_dir}}/drpc.log" + }, + { + "type":"storm_logviewer", + "rowtype":"service", + "path":"{{storm_log_dir}}/logviewer.log" + }, + { + "type":"storm_nimbus", + "rowtype":"service", + "path":"{{storm_log_dir}}/nimbus.log" + }, + { + "type":"storm_supervisor", + "rowtype":"service", + "path":"{{storm_log_dir}}/supervisor.log" + }, + { + "type":"storm_ui", + "rowtype":"service", + "path":"{{storm_log_dir}}/ui.log" + }, + { + "type":"storm_worker", + "rowtype":"service", + "path":"{{storm_log_dir}}/*worker*.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "storm_drpc", + "storm_logviewer", + "storm_nimbus", + "storm_supervisor", + "storm_ui", + "storm_worker" + ] + + } + + }, + "log4j_format":"", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss.SSS" + } + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2 new file mode 100644 index 0000000..fb35620 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2 @@ -0,0 +1,86 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"mapred_historyserver", + "rowtype":"service", + "path":"{{mapred_log_dir_prefix}}/mapred/mapred-mapred-historyserver*.log" + }, + { + "type":"yarn_nodemanager", + "rowtype":"service", + "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-nodemanager-*.log" + }, + { + "type":"yarn_resourcemanager", + "rowtype":"service", + "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-resourcemanager-*.log" + }, + { + "type":"yarn_timelineserver", + "rowtype":"service", + "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-timelineserver-*.log" + }, + { + "type":"yarn_historyserver", + "rowtype":"service", + "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-historyserver-*.log" + }, + { + "type":"yarn_jobsummary", + "rowtype":"service", + "path":"{{yarn_log_dir_prefix}}/yarn/hadoop-mapreduce.jobsummary.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "mapred_historyserver", + "yarn_historyserver", + "yarn_jobsummary", + "yarn_nodemanager", + "yarn_resourcemanager", + "yarn_timelineserver" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zookeeper.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zookeeper.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zookeeper.json.j2 new file mode 100644 index 0000000..083bd8a --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zookeeper.json.j2 @@ -0,0 +1,56 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"zookeeper", + "rowtype":"service", + "path":"{{zk_log_dir}}/zookeeper*.out" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "zookeeper" + ] + + } + + }, + "log4j_format":"%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}-%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\@%{INT:line_number}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logfeeder.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logfeeder.properties.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logfeeder.properties.j2 new file mode 100644 index 0000000..1edf16a --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logfeeder.properties.j2 @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +logfeeder.checkpoint.folder={{logfeeder_checkpoint_folder}} +metrics.collector.hosts={{metrics_collector_hosts}} +config.files={{logfeeder_config_files}} http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2 new file mode 100644 index 0000000..1e183bd --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2 @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#solr.url=http://solr_host:{{logsearch_solr_port}}/solr +solr.zkhosts={{zookeeper_quorum}}{{logsearch_solr_znode}} + +# Service Logs +solr.core.logs={{logsearch_solr_collection_service_logs}} + +solr.service_logs.split_interval_mins={{service_logs_collection_splits_interval_mins}} +solr.service_logs.shards={{logsearch_numshards}} +solr.service_logs.replication_factor={{logsearch_repfactor}} + +solr.servicelogs.fields={{logsearch_service_logs_fields}} + +# Audit logs +auditlog.solr.zkhosts={{solr_audit_logs_zk_quorum}}{{solr_audit_logs_zk_node}} +auditlog.solr.core.logs={{logsearch_solr_collection_audit_logs}} +auditlog.solr.url={{solr_audit_logs_url}} + +solr.audit_logs.split_interval_mins={{audit_logs_collection_splits_interval_mins}} +solr.audit_logs.shards={{logsearch_numshards}} +solr.audit_logs.replication_factor={{logsearch_repfactor}} + +# History logs +solr.core.history=history +solr.history.config_name=history +solr.history.replication_factor={{logsearch_repfactor}} http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/output.config.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/output.config.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/output.config.json.j2 new file mode 100644 index 0000000..8216587 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/output.config.json.j2 @@ -0,0 +1,97 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "output":[ + { + "is_enabled":"{{solr_service_logs_enable}}", + "comment":"Output to solr for service logs", + "destination":"solr", + "zk_hosts":"{{zookeeper_quorum}}{{logsearch_solr_znode}}", + "collection":"{{logsearch_solr_collection_service_logs}}", + "number_of_shards": "{{logsearch_numshards}}", + "splits_interval_mins": "{{service_logs_collection_splits_interval_mins}}", + "conditions":{ + "fields":{ + "rowtype":[ + "service" + ] + + } + + } + + }, + { + "comment":"Output to solr for audit records", + "is_enabled":"{{solr_audit_logs_enable}}", + "destination":"solr", + "zk_hosts":"{{zookeeper_quorum}}{{logsearch_solr_znode}}", + "collection":"{{logsearch_solr_collection_audit_logs}}", + "number_of_shards": "{{logsearch_numshards}}", + "splits_interval_mins": "{{audit_logs_collection_splits_interval_mins}}", + "conditions":{ + "fields":{ + "rowtype":[ + "audit" + ] + + } + + } + + }, + { + "is_enabled":"{{kafka_service_logs_enable}}", + "destination":"kafka", + "broker_list":"{{kafka_broker_list}}", + "topic":"{{kafka_topic_service_logs}}", + "kafka.security.protocol":"{{kafka_security_protocol}}", + "kafka.sasl.kerberos.service.name":"{{kafka_kerberos_service_name}}", + "conditions":{ + "fields":{ + "rowtype":[ + "service" + ] + + } + + } + + }, + { + "is_enabled":"{{kafka_audit_logs_enable}}", + "destination":"kafka", + "broker_list":"{{kafka_broker_list}}", + "topic":"{{kafka_topic_audit_logs}}", + "kafka.security.protocol":"{{kafka_security_protocol}}", + "kafka.sasl.kerberos.service.name":"{{kafka_kerberos_service_name}}", + "conditions":{ + "fields":{ + "rowtype":[ + "audit" + ] + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/zoo.cfg.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/zoo.cfg.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/zoo.cfg.j2 new file mode 100644 index 0000000..1f3808b --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/zoo.cfg.j2 @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{% for key, value in zoo_cfg_properties_map.iteritems() -%} + {{key}}={{value}} +{% endfor %} +{% for host in zookeeper_hosts_list -%} +server.{{loop.index}}={{host}}:2888:3888 +{% endfor %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/quicklinks/quicklinks.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/quicklinks/quicklinks.json b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/quicklinks/quicklinks.json new file mode 100644 index 0000000..48a3efe --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/quicklinks/quicklinks.json @@ -0,0 +1,26 @@ +{ + "name": "default", + "description": "default quick links configuration", + "configuration": { + "protocol": { + "type": "http" + }, + "links": [ + { + "name": "logsearch_ui", + "label": "Logsearch UI", + "url": "%@://%@:%@", + "requires_user_name": "false", + "port": { + "http_property": "logsearch.ui.port", + "http_default_port": "8888", + "https_property": "logsearch.ui.port", + "https_default_port": "8888", + "regex": "^(\\d+)$", + "site": "logsearch-site" + } + } + ] + } +} + http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml index 77ff982..b0cff68 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml @@ -174,6 +174,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -210,6 +211,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -322,6 +324,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml index 0ef4674..bf041de 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml @@ -180,6 +180,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -216,6 +217,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -328,6 +330,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/main/resources/stacks/HDP/2.3/services/LOGSEARCH/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/LOGSEARCH/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/LOGSEARCH/metainfo.xml new file mode 100644 index 0000000..df697dc --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/LOGSEARCH/metainfo.xml @@ -0,0 +1,26 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>LOGSEARCH</name> + <extends>common-services/LOGSEARCH/0.5.0</extends> + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/main/resources/stacks/HDP/2.3/services/LOGSEARCH/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/LOGSEARCH/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.3/services/LOGSEARCH/role_command_order.json new file mode 100755 index 0000000..688186b --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/LOGSEARCH/role_command_order.json @@ -0,0 +1,8 @@ +{ + "general_deps" : { + "_comment" : "dependencies for all logsearch", + "LOGSEARCH_SOLR-START" : ["ZOOKEEPER_SERVER-START"], + "LOGSEARCH_SERVER-START": ["LOGSEARCH_SOLR-START"], + "LOGSEARCH_LOGFEEDER-START": ["LOGSEARCH_SOLR-START", "LOGSEARCH_SERVER-START"] + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml index ed30846..6b74af0 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml @@ -175,6 +175,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -213,6 +214,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -339,6 +341,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml index 4731631..9fb2bba 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml @@ -166,6 +166,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -203,6 +204,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -320,6 +322,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml index 4a6241d..4c77461 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml @@ -172,6 +172,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -209,6 +210,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -326,6 +328,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml index b53a090..e3bc7a3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml @@ -173,6 +173,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -211,6 +212,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -337,6 +339,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml index c9ea512..1e93573 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml @@ -166,6 +166,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -203,6 +204,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -320,6 +322,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml index 5503c1a..2acffda 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml @@ -173,6 +173,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -211,6 +212,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> @@ -337,6 +339,7 @@ </priority> <exclude> <service>AMBARI_METRICS</service> + <service>LOGSEARCH</service> </exclude> </group> http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java index d7a15e2..8e9f81a 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java @@ -794,4 +794,66 @@ public class StackManagerTest { assertTrue(rangerUserSyncRoleCommand + " should be dependent of " + kmsRoleCommand, rangerUserSyncBlockers.contains(kmsRoleCommand)); } //todo: component override assertions + + @Test + public void testServicesWithLogsearchRoleCommandOrder() throws AmbariException { + URL rootDirectoryURL = StackManagerTest.class.getResource("/"); + Assert.notNull(rootDirectoryURL); + + File resourcesDirectory = new File(new File(rootDirectoryURL.getFile()).getParentFile().getParentFile(), "src/main/resources"); + + File stackRoot = new File(resourcesDirectory, "stacks"); + File commonServices = new File(resourcesDirectory, "common-services"); + + MetainfoDAO metaInfoDao = createNiceMock(MetainfoDAO.class); + StackDAO stackDao = createNiceMock(StackDAO.class); + ActionMetadata actionMetadata = createNiceMock(ActionMetadata.class); + Configuration config = createNiceMock(Configuration.class); + + expect(config.getSharedResourcesDirPath()).andReturn( + ClassLoader.getSystemClassLoader().getResource("").getPath()).anyTimes(); + + replay(config, metaInfoDao, stackDao, actionMetadata); + + OsFamily osFamily = new OsFamily(config); + + StackManager stackManager = new StackManager(stackRoot, commonServices, osFamily, metaInfoDao, actionMetadata, stackDao); + + String zookeeperServerRoleCommand = Role.ZOOKEEPER_SERVER + "-" + RoleCommand.START; + String logsearchServerRoleCommand = Role.LOGSEARCH_SERVER + "-" + RoleCommand.START; + String logsearchSolrRoleCommand = Role.LOGSEARCH_SOLR + "-" + RoleCommand.START; + String logsearchLogfeederRoleCommand = Role.LOGSEARCH_LOGFEEDER + "-" + RoleCommand.START; + + StackInfo hdp = stackManager.getStack("HDP", "2.3"); + Map<String, Object> rco = hdp.getRoleCommandOrder().getContent(); + + Map<String, Object> generalDeps = (Map<String, Object>) rco.get("general_deps"); + Map<String, Object> optionalNoGlusterfs = (Map<String, Object>) rco.get("optional_no_glusterfs"); + + // HDFS/YARN - verify that the stack level rco still works as expected + String nameNodeRoleCommand = Role.NAMENODE + "-" + RoleCommand.START; + String rangerUserSyncRoleCommand = Role.RANGER_USERSYNC + "-" + RoleCommand.START; + ArrayList<String> nameNodeBlockers = (ArrayList<String>) optionalNoGlusterfs.get(nameNodeRoleCommand); + + assertTrue(nameNodeRoleCommand + " should be dependent of " + rangerUserSyncRoleCommand, nameNodeBlockers.contains(rangerUserSyncRoleCommand)); + + String resourceManagerCommandRoleCommand = Role.RESOURCEMANAGER + "-" + RoleCommand.START; + ArrayList<String> resourceManagerBlockers = (ArrayList<String>)generalDeps.get(resourceManagerCommandRoleCommand); + + assertTrue(resourceManagerCommandRoleCommand + " should be dependent of " + rangerUserSyncRoleCommand, resourceManagerBlockers.contains(rangerUserSyncRoleCommand)); + + // verify logsearch rco + // LogSearch Solr + ArrayList<String> logsearchSolrBlockers = (ArrayList<String>) generalDeps.get(logsearchSolrRoleCommand); + assertTrue(logsearchSolrRoleCommand + " should be dependent of " + zookeeperServerRoleCommand, logsearchSolrBlockers.contains(zookeeperServerRoleCommand)); + + // LogSearch Server + ArrayList<String> logsearchServerBlockers = (ArrayList<String>) generalDeps.get(logsearchServerRoleCommand); + assertTrue(logsearchServerRoleCommand + " should be dependent of " + logsearchSolrRoleCommand, logsearchServerBlockers.contains(logsearchSolrRoleCommand)); + + // LogSearch LogFeeder + ArrayList<String> logsearchLogfeederBlockers = (ArrayList<String>) generalDeps.get(logsearchLogfeederRoleCommand); + assertTrue(logsearchLogfeederRoleCommand + " should be dependent of " + logsearchSolrRoleCommand, logsearchLogfeederBlockers.contains(logsearchSolrRoleCommand)); + assertTrue(logsearchLogfeederRoleCommand + " should be dependent of " + logsearchServerRoleCommand, logsearchLogfeederBlockers.contains(logsearchServerRoleCommand)); + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py new file mode 100644 index 0000000..f485d27 --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +from stacks.utils.RMFTestCase import RMFTestCase, Template, InlineTemplate, StaticFile +from resource_management.core.exceptions import ComponentIsNotRunning + + +class TestLogFeeder(RMFTestCase): + COMMON_SERVICES_PACKAGE_DIR = "LOGSEARCH/0.5.0/package" + STACK_VERSION = "2.4" + + def configureResourcesCalled(self): + self.assertResourceCalled('Directory', '/var/log/ambari-logsearch-logfeeder', + owner='logfeeder', + group='logfeeder', + create_parents=True, + cd_access='a', + mode=0755 + ) + self.assertResourceCalled('Directory', '/var/run/ambari-logsearch-logfeeder', + owner='logfeeder', + group='logfeeder', + create_parents=True, + cd_access='a', + mode=0755 + ) + self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-logfeeder', + owner='logfeeder', + group='logfeeder', + create_parents=True, + cd_access='a', + mode=0755 + ) + self.assertResourceCalled('Directory', '/etc/ambari-logsearch-logfeeder/conf', + owner='logfeeder', + group='logfeeder', + create_parents=True, + cd_access='a', + mode=0755 + ) + self.assertResourceCalled('Directory', '/etc/ambari-logsearch-logfeeder/conf/checkpoints', + owner='logfeeder', + group='logfeeder', + create_parents=True, + cd_access='a', + mode=0755 + ) + + self.assertResourceCalled('File', '/var/log/ambari-logsearch-logfeeder/logfeeder.out', + owner='logfeeder', + group='logfeeder', + mode=0644, + content='' + ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/logfeeder.properties', + owner='logfeeder', + content=Template('logfeeder.properties.j2') + ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/logfeeder-env.sh', + owner='logfeeder', + content=InlineTemplate(self.getConfig()['configurations']['logfeeder-env']['content']) + ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/log4j.xml', + owner='logfeeder', + content=InlineTemplate(self.getConfig()['configurations']['logfeeder-log4j']['content']) + ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/grok-patterns', + owner='logfeeder', + content=Template('grok-patterns.j2'), + encoding='utf-8' + ) + + logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'hbase', 'hdfs', 'hive', 'kafka', + 'knox', 'logsearch', 'oozie', 'ranger', 'storm', 'yarn', 'zookeeper'] + + logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag + in logfeeder_supported_services] + + for file_name in logfeeder_config_file_names: + self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/' + file_name, + owner='logfeeder', + content=Template(file_name + ".j2") + ) + + def test_configure_default(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logfeeder.py", + classname="LogFeeder", + command="configure", + config_file="default.json", + stack_version=self.STACK_VERSION, + target=RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.configureResourcesCalled() + self.assertNoMoreResources() + + def test_start_default(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logfeeder.py", + classname="LogFeeder", + command="start", + config_file="default.json", + stack_version=self.STACK_VERSION, + target=RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.configureResourcesCalled() + self.assertResourceCalled('Execute', '/usr/lib/ambari-logsearch-logfeeder/run.sh', + environment={ + 'LOGFEEDER_INCLUDE': '/etc/ambari-logsearch-logfeeder/conf/logfeeder-env.sh'}, + user='logfeeder' + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py new file mode 100644 index 0000000..46517f0 --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +from stacks.utils.RMFTestCase import RMFTestCase, Template, InlineTemplate, StaticFile +from resource_management.core.exceptions import ComponentIsNotRunning +from mock.mock import MagicMock, patch +from resource_management.libraries.script.config_dictionary import UnknownConfiguration + +class TestLogSearch(RMFTestCase): + COMMON_SERVICES_PACKAGE_DIR = "LOGSEARCH/0.5.0/package" + STACK_VERSION = "2.4" + + def configureResourcesCalled(self): + self.assertResourceCalled('Directory', '/var/log/ambari-logsearch-portal', + owner = 'logsearch', + group = 'logsearch', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + self.assertResourceCalled('Directory', '/var/run/ambari-logsearch-portal', + owner = 'logsearch', + group = 'logsearch', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-portal', + owner = 'logsearch', + group = 'logsearch', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + self.assertResourceCalled('Directory', '/etc/ambari-logsearch-portal/conf', + owner = 'logsearch', + group = 'logsearch', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + + self.assertResourceCalled('File', '/var/log/ambari-logsearch-portal/logsearch.out', + owner = 'logsearch', + group = 'logsearch', + mode = 0644, + content = '' + ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/logsearch.properties', + owner = 'logsearch', + content = Template('logsearch.properties.j2') + ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/log4j.xml', + owner = 'logsearch', + content = InlineTemplate(self.getConfig()['configurations']['logsearch-log4j']['content']) + ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/logsearch-env.sh', + content = InlineTemplate(self.getConfig()['configurations']['logsearch-env']['content']), + mode = 0755, + owner = "logsearch" + ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/solr_configsets/hadoop_logs/conf/solrconfig.xml', + owner = 'logsearch', + content = InlineTemplate(self.getConfig()['configurations']['logsearch-service_logs-solrconfig']['content']) + ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/solr_configsets/audit_logs/conf/solrconfig.xml', + owner = 'logsearch', + content = InlineTemplate(self.getConfig()['configurations']['logsearch-audit_logs-solrconfig']['content']) + ) + + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:None/logsearch -cmd downconfig -confdir /tmp/solr_config_hadoop_logs_0.[0-9]* -confname hadoop_logs$', + only_if = "^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:None/logsearch -cmd get /configs/hadoop_logs$" + ) + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:None/logsearch -cmd upconfig -confdir /etc/ambari-logsearch-portal/conf/solr_configsets/hadoop_logs/conf -confname hadoop_logs$', + not_if = "^test -d /tmp/solr_config_hadoop_logs_0.[0-9]*$" + ) + + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:None/logsearch -cmd downconfig -confdir /tmp/solr_config_history_0.[0-9]* -confname history$', + only_if = "^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:None/logsearch -cmd get /configs/history$" + ) + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:None/logsearch -cmd upconfig -confdir /etc/ambari-logsearch-portal/conf/solr_configsets/history/conf -confname history$', + not_if = "^test -d /tmp/solr_config_history_0.[0-9]*$" + ) + + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:None/logsearch -cmd downconfig -confdir /tmp/solr_config_audit_logs_0.[0-9]* -confname audit_logs$', + only_if = "^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:None/logsearch -cmd get /configs/audit_logs$" + ) + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:None/logsearch -cmd upconfig -confdir /etc/ambari-logsearch-portal/conf/solr_configsets/audit_logs/conf -confname audit_logs$', + not_if = "^test -d /tmp/solr_config_audit_logs_0.[0-9]*$" + ) + self.assertResourceCalled('Execute', ('chmod', '-R', 'ugo+r', '/etc/ambari-logsearch-portal/conf/solr_configsets'), + sudo=True + ) + + def test_configure_default(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logsearch.py", + classname = "LogSearch", + command = "configure", + config_file = "default.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.configureResourcesCalled() + self.assertNoMoreResources() + + def test_start_default(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logsearch.py", + classname = "LogSearch", + command = "start", + config_file = "default.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.configureResourcesCalled() + self.assertResourceCalled('Execute', "/usr/lib/ambari-logsearch-portal/run.sh 61888", + environment = {'LOGSEARCH_INCLUDE': '/etc/ambari-logsearch-portal/conf/logsearch-env.sh'}, + user = "logsearch" + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/f903d17f/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_solr.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_solr.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_solr.py new file mode 100644 index 0000000..977cd9c --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_solr.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +from stacks.utils.RMFTestCase import RMFTestCase, Template, InlineTemplate, StaticFile +from resource_management.core.exceptions import ComponentIsNotRunning +from resource_management.libraries.script.config_dictionary import UnknownConfiguration + +class TestSolr(RMFTestCase): + COMMON_SERVICES_PACKAGE_DIR = "LOGSEARCH/0.5.0/package" + STACK_VERSION = "2.4" + + def configureResourcesCalled(self): + self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-solr', + owner = 'solr', + group = 'solr', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + self.assertResourceCalled('Directory', '/var/log/ambari-logsearch-solr', + owner = 'solr', + group = 'solr', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + self.assertResourceCalled('Directory', '/var/run/ambari-logsearch-solr', + owner = 'solr', + group = 'solr', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + self.assertResourceCalled('Directory', '/etc/ambari-logsearch-solr/conf', + owner = 'solr', + group = 'solr', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + self.assertResourceCalled('Directory', '/opt/logsearch_solr/data', + owner = 'solr', + group = 'solr', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + self.assertResourceCalled('Directory', '/opt/logsearch_solr/data/resources', + owner = 'solr', + group = 'solr', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + + self.assertResourceCalled('File', '/var/log/ambari-logsearch-solr/solr-install.log', + owner = 'solr', + group = 'solr', + mode = 0644, + content = '' + ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-solr/conf/logsearch-solr-env.sh', + owner = 'solr', + mode = 0755, + content = InlineTemplate(self.getConfig()['configurations']['logsearch-solr-env']['content']) + ) + self.assertResourceCalled('File', '/opt/logsearch_solr/data/solr.xml', + owner = 'solr', + content = InlineTemplate(self.getConfig()['configurations']['logsearch-solr-xml']['content']) + ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-solr/conf/log4j.properties', + owner = 'solr', + content = InlineTemplate(self.getConfig()['configurations']['logsearch-solr-log4j']['content']) + ) + self.assertResourceCalled('File', '/opt/logsearch_solr/data/zoo.cfg', + owner = 'solr', + content = Template('zoo.cfg.j2') + ) + self.assertResourceCalled('Execute', 'export JAVA_HOME=/usr/jdk64/jdk1.7.0_45; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org -cmd makepath /logsearch', + not_if = "export JAVA_HOME=/usr/jdk64/jdk1.7.0_45; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org -cmd get /logsearch", + ignore_failures = True, + user = "solr" + ) + + def test_configure_default(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/solr.py", + classname = "Solr", + command = "configure", + config_file = "default.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.configureResourcesCalled() + self.assertNoMoreResources() + + def test_start_default(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/solr.py", + classname = "Solr", + command = "start", + config_file = "default.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.configureResourcesCalled() + self.assertResourceCalled('Execute', "/usr/lib/ambari-logsearch-solr/bin/solr start -cloud -noprompt -s /opt/logsearch_solr/data >> /var/log/ambari-logsearch-solr/solr-install.log 2>&1", + environment = {'SOLR_INCLUDE': '/etc/ambari-logsearch-solr/conf/logsearch-solr-env.sh'}, + user = "solr" + ) + + def test_stop_default(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/solr.py", + classname = "Solr", + command = "stop", + config_file = "default.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + self.assertResourceCalled('Execute', '/usr/lib/ambari-logsearch-solr/bin/solr stop -all >> /var/log/ambari-logsearch-solr/solr-install.log', + environment = {'SOLR_INCLUDE': '/etc/ambari-logsearch-solr/conf/logsearch-solr-env.sh'}, + user = "solr", + only_if = "test -f /var/run/ambari-logsearch-solr/solr-8886.pid" + ) + self.assertResourceCalled('File', '/var/run/ambari-logsearch-solr/solr-8886.pid', + action = ['delete'] + )
