http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/hadoop/metrics2/sink/timeline/configuration/Configuration.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/hadoop/metrics2/sink/timeline/configuration/Configuration.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/hadoop/metrics2/sink/timeline/configuration/Configuration.java new file mode 100644 index 0000000..a0380e1 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/hadoop/metrics2/sink/timeline/configuration/Configuration.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.sink.timeline.configuration; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class Configuration { + public final Log LOG = LogFactory.getLog(this.getClass()); + private final Properties properties; + + public Configuration(String configFile) { + properties = new Properties(); + + //Get property file stream from classpath + InputStream inputStream = Configuration.class.getResourceAsStream(configFile); + + if (inputStream == null) { + throw new IllegalArgumentException(configFile + " not found in classpath"); + } + + // load the properties + try { + properties.load(inputStream); + inputStream.close(); + } catch (FileNotFoundException fnf) { + LOG.info("No configuration file " + configFile + " found in classpath.", fnf); + } catch (IOException ie) { + throw new IllegalArgumentException("Can't read configuration file " + + configFile, ie); + } + } + + public String getProperty(String key) { + return properties.getProperty(key); + } + + public String getProperty(String key, String defaultValue) { + return properties.getProperty(key, defaultValue); + } +}
http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/hadoop/metrics2/sink/util/Servers.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/hadoop/metrics2/sink/util/Servers.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/hadoop/metrics2/sink/util/Servers.java new file mode 100644 index 0000000..b3dc46f --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/hadoop/metrics2/sink/util/Servers.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.metrics2.sink.util; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.List; + +/** + * Helpers to handle server addresses + */ +public class Servers { + /** + * This class is not intended to be instantiated + */ + private Servers() {} + + /** + * Parses a space and/or comma separated sequence of server specifications + * of the form <i>hostname</i> or <i>hostname:port</i>. If + * the specs string is null, defaults to localhost:defaultPort. + * + * @param specs server specs (see description) + * @param defaultPort the default port if not specified + * @return a list of InetSocketAddress objects. + */ + public static List<InetSocketAddress> parse(String specs, int defaultPort) { + List<InetSocketAddress> result = new ArrayList<InetSocketAddress>(); + if (specs == null) { + result.add(new InetSocketAddress("localhost", defaultPort)); + } else { + String[] specStrings = specs.split("[ ,]+"); + for (String specString : specStrings) { + result.add(createSocketAddr(specString, defaultPort)); + } + } + return result; + } + + /** + * @param host + * @param port + * @return a InetSocketAddress created with the specified host and port + */ + private static InetSocketAddress createSocketAddr(String target, int defaultPort) { + String helpText = ""; + if (target == null) { + throw new IllegalArgumentException("Target address cannot be null." + helpText); + } + boolean hasScheme = target.contains("://"); + URI uri = null; + try { + uri = hasScheme ? URI.create(target) : URI.create("dummyscheme://" + target); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Does not contain a valid host:port authority: " + target + helpText); + } + + String host = uri.getHost(); + int port = uri.getPort(); + if (port == -1) { + port = defaultPort; + } + String path = uri.getPath(); + + if ((host == null) || (port < 0) || (!hasScheme && path != null && !path.isEmpty())) { + throw new IllegalArgumentException("Does not contain a valid host:port authority: " + target + helpText); + } + return createSocketAddrForHost(host, port); + } + + /** + * @param host + * @param port + * @return a InetSocketAddress created with the specified host and port + */ + private static InetSocketAddress createSocketAddrForHost(String host, int port) { + InetSocketAddress addr; + try { + InetAddress iaddr = InetAddress.getByName(host); + iaddr = InetAddress.getByAddress(host, iaddr.getAddress()); + addr = new InetSocketAddress(iaddr, port); + } catch (UnknownHostException e) { + addr = InetSocketAddress.createUnresolved(host, port); + } + return addr; + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/control ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/control b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/control new file mode 100644 index 0000000..40cd855 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/control @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +Package: [[artifactId]] +Version: [[package-version]]-[[package-release]] +Section: [[deb.section]] +Priority: [[deb.priority]] +Depends: [[deb.dependency.list]] +Architecture: [[deb.architecture]] +Description: [[description]] +Maintainer: [[deb.publisher]] http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postinst ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postinst b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postinst new file mode 100644 index 0000000..21a01fa --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postinst @@ -0,0 +1,15 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postrm ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postrm b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postrm new file mode 100644 index 0000000..21a01fa --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postrm @@ -0,0 +1,15 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/preinst ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/preinst b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/preinst new file mode 100644 index 0000000..21a01fa --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/preinst @@ -0,0 +1,15 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/prerm ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/prerm b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/prerm new file mode 100644 index 0000000..21a01fa --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/prerm @@ -0,0 +1,15 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/alias_config.json ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/alias_config.json b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/alias_config.json new file mode 100644 index 0000000..d8a239a --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/alias_config.json @@ -0,0 +1,42 @@ +{ + "input": { + "file": { + "klass": "org.apache.ambari.logfeeder.input.InputFile" + } + + }, + "filter": { + "json": { + "klass": "org.apache.ambari.logfeeder.filter.JSONFilterCode" + }, + "keyvalue": { + "klass": "org.apache.ambari.logfeeder.filter.FilterKeyValue" + }, + "grok": { + "klass": "org.apache.ambari.logfeeder.filter.FilterGrok" + } + }, + + "mapper": { + "map_date": { + "klass": "org.apache.ambari.logfeeder.mapper.MapperDate" + }, + "map_fieldname": { + "klass": "org.apache.ambari.logfeeder.mapper.MapperFieldName" + }, + "map_fieldvalue": { + "klass": "org.apache.ambari.logfeeder.mapper.MapperFieldValue" + } + }, + "output": { + "solr": { + "klass": "org.apache.ambari.logfeeder.output.OutputSolr" + }, + "file": { + "klass": "org.apache.ambari.logfeeder.output.OutputFile" + }, + "kafka": { + "klass": "org.apache.ambari.logfeeder.output.OutputKafka" + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/config.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/config.json.j2 b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/config.json.j2 new file mode 100644 index 0000000..163ee2b --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/config.json.j2 @@ -0,0 +1,995 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "global":{ + "add_fields":{ + "cluster":"{{cluster_name}}" + }, + "source":"file", + "tail":"true", + "gen_event_md5":"true", + "start_position":"beginning" + }, + "input":[ + { + "type":"accumulo_gc", + "rowtype":"service", + "path":"{{accumulo_log_dir}}/gc_*.log" + }, + { + "type":"accumulo_master", + "rowtype":"service", + "path":"{{accumulo_log_dir}}/master_*.log" + }, + { + "type":"accumulo_monitor", + "rowtype":"service", + "path":"{{accumulo_log_dir}}/monitor_*.log" + }, + { + "type":"accumulo_tracer", + "rowtype":"service", + "path":"{{accumulo_log_dir}}/tracer_*.log" + }, + { + "type":"accumulo_tserver", + "rowtype":"service", + "path":"{{accumulo_log_dir}}/tserver_*.log" + }, + { + "type":"atlas_app", + "rowtype":"service", + "path":"{{atlas_log_dir}}/application.log" + }, + { + "type":"ambari_agent", + "rowtype":"service", + "path":"{{ambari_agent_log_dir}}/ambari-agent.log" + }, + { + "type":"ambari_server", + "rowtype":"service", + "path":"{{ambari_server_log_dir}}/ambari-server.log" + }, + { + "type":"ams_hbase_master", + "rowtype":"service", + "path":"{{metrics_collector_log_dir}}/hbase-ams-master-*.log" + }, + { + "type":"ams_hbase_regionserver", + "rowtype":"service", + "path":"{{metrics_collector_log_dir}}/hbase-ams-regionserver-*.log" + }, + { + "type":"ams_collector", + "rowtype":"service", + "path":"{{metrics_collector_log_dir}}/ambari-metrics-collector.log" + }, + { + "type":"falcon_app", + "rowtype":"service", + "path":"{{falcon_log_dir}}/falcon.application.log" + }, + { + "type":"hbase_master", + "rowtype":"service", + "path":"{{hbase_log_dir}}/hbase-hbase-master-*.log" + }, + { + "type":"hbase_regionserver", + "rowtype":"service", + "path":"{{hbase_log_dir}}/hbase-hbase-regionserver-*.log" + }, + { + "type":"hdfs_datanode", + "rowtype":"service", + "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-datanode-*.log" + }, + { + "type":"hdfs_namenode", + "rowtype":"service", + "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-namenode-*.log" + }, + { + "type":"hdfs_journalnode", + "rowtype":"service", + "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-journalnode-*.log" + }, + { + "type":"hdfs_secondarynamenode", + "rowtype":"service", + "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-secondarynamenode-*.log" + }, + { + "type":"hdfs_zkfc", + "rowtype":"service", + "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-zkfc-*.log" + }, + { + "type":"hive_hiveserver2", + "rowtype":"service", + "path":"{{hive_log_dir}}/hiveserver2.log" + }, + { + "type":"hive_metastore", + "rowtype":"service", + "path":"{{hive_log_dir}}/hivemetastore.log" + }, + { + "type":"kafka_controller", + "rowtype":"service", + "path":"{{kafka_log_dir}}/controller.log" + }, + { + "type":"kafka_request", + "rowtype":"service", + "path":"{{kafka_log_dir}}/kafka-request.log" + }, + { + "type":"kafka_logcleaner", + "rowtype":"service", + "path":"{{kafka_log_dir}}/log-cleaner.log" + }, + { + "type":"kafka_server", + "rowtype":"service", + "path":"{{kafka_log_dir}}/server.log" + }, + { + "type":"kafka_statechange", + "rowtype":"service", + "path":"{{kafka_log_dir}}/state-change.log" + }, + { + "type":"knox_gateway", + "rowtype":"service", + "path":"{{knox_log_dir}}/gateway.log" + }, + { + "type":"knox_cli", + "rowtype":"service", + "path":"{{knox_log_dir}}/knoxcli.log" + }, + { + "type":"knox_ldap", + "rowtype":"service", + "path":"{{knox_log_dir}}/ldap.log" + }, + { + "type":"mapred_historyserver", + "rowtype":"service", + "path":"{{mapred_log_dir_prefix}}/mapred/mapred-mapred-historyserver*.log" + }, + { + "type":"logsearch_app", + "rowtype":"service", + "path":"{{logsearch_log_dir}}/logsearch.log" + }, + { + "type":"logsearch_feeder", + "rowtype":"service", + "path":"{{logfeeder_log_dir}}/logfeeder.log" + }, + { + "type":"logsearch_perf", + "rowtype":"service", + "path":"{{logsearch_log_dir}}/logsearch-performance.log" + }, + { + "type":"ranger_admin", + "rowtype":"service", + "path":"{{ranger_admin_log_dir}}/xa_portal.log" + }, + { + "type":"ranger_dbpatch", + "is_enabled":"true", + "path":"{{ranger_admin_log_dir}}/ranger_db_patch.log" + }, + { + "type":"ranger_kms", + "rowtype":"service", + "path":"{{ranger_kms_log_dir}}/kms.log" + }, + { + "type":"ranger_usersync", + "rowtype":"service", + "path":"{{ranger_usersync_log_dir}}/usersync.log" + }, + { + "type":"oozie_app", + "rowtype":"service", + "path":"{{oozie_log_dir}}/oozie.log" + }, + { + "type":"yarn_nodemanager", + "rowtype":"service", + "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-nodemanager-*.log" + }, + { + "type":"yarn_resourcemanager", + "rowtype":"service", + "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-resourcemanager-*.log" + }, + { + "type":"yarn_timelineserver", + "rowtype":"service", + "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-timelineserver-*.log" + }, + { + "type":"yarn_historyserver", + "rowtype":"service", + "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-historyserver-*.log" + }, + { + "type":"yarn_jobsummary", + "rowtype":"service", + "path":"{{yarn_log_dir_prefix}}/yarn/hadoop-mapreduce.jobsummary.log" + }, + { + "type":"storm_drpc", + "rowtype":"service", + "path":"{{storm_log_dir}}/drpc.log" + }, + { + "type":"storm_logviewer", + "rowtype":"service", + "path":"{{storm_log_dir}}/logviewer.log" + }, + { + "type":"storm_nimbus", + "rowtype":"service", + "path":"{{storm_log_dir}}/nimbus.log" + }, + { + "type":"storm_supervisor", + "rowtype":"service", + "path":"{{storm_log_dir}}/supervisor.log" + }, + { + "type":"storm_ui", + "rowtype":"service", + "path":"{{storm_log_dir}}/ui.log" + }, + { + "type":"storm_worker", + "rowtype":"service", + "path":"{{storm_log_dir}}/*worker*.log" + }, + { + "type":"zookeeper", + "rowtype":"service", + "path":"{{zk_log_dir}}/zookeeper/zookeeper*.out" + }, + { + "type":"hdfs_audit", + "rowtype":"audit", + "is_enabled":"true", + "add_fields":{ + "logType":"HDFSAudit", + "enforcer":"hadoop-acl", + "repoType":"1", + "repo":"hdfs" + }, + "path":"{{hdfs_log_dir_prefix}}/hdfs/hdfs-audit.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "accumulo_master" + ] + + } + + }, + "log4j_format":"%d{ISO8601} [%-8c{2}] %-5p: %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "comment":"This one has one extra space after LEVEL", + "conditions":{ + "fields":{ + "type":[ + "accumulo_gc", + "accumulo_monitor", + "accumulo_tracer", + "accumulo_tserver" + ] + + } + + }, + "log4j_format":"%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "atlas_app", + "falcon_app" + ] + + } + + }, + "log4j_format":"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{SPACE}-%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}~%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ams_collector" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %p %c: %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ams_hbase_master", + "ams_hbase_regionserver", + "hbase_master", + "hbase_regionserver" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_agent" + ] + + } + + }, + "log4j_format":"", + "multiline_pattern":"^(%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime} %{JAVAFILE:file}:%{INT:line_number} - %{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + }, + "level":{ + "map_fieldvalue":{ + "pre_value":"WARNING", + "post_value":"WARN" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_server" + ] + + } + + }, + "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n", + "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})", + "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"dd MMM yyyy HH:mm:ss" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "hdfs_datanode", + "hdfs_journalnode", + "hdfs_secondarynamenode", + "hdfs_namenode", + "hdfs_zkfc", + "knox_gateway", + "knox_cli", + "knox_ldap", + "mapred_historyserver", + "yarn_historyserver", + "yarn_jobsummary", + "yarn_nodemanager", + "yarn_resourcemanager", + "yarn_timelineserver" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "hive_hiveserver2", + "hive_metastore" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]:%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "kafka_controller", + "kafka_request", + "kafka_logcleaner" + ] + + } + + }, + "log4j_format":"[%d] %p %m (%c)%n", + "multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])", + "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "comment":"Suppose to be same log4j pattern as other kafka processes, but some reason thread is not printed", + "conditions":{ + "fields":{ + "type":[ + "kafka_server", + "kafka_statechange" + ] + + } + + }, + "log4j_format":"[%d] %p %m (%c)%n", + "multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])", + "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "oozie_app" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %5p %c{1}:%L - SERVER[${oozie.instance.id}] %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{DATA:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "logsearch_app", + "logsearch_feeder", + "logsearch_perf", + "ranger_admin", + "ranger_dbpatch" + ] + + } + + }, + "log4j_format":"%d [%t] %-5p %C{6} (%F:%L) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ranger_kms" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p %c{1} - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ranger_usersync" + ] + + } + + }, + "log4j_format":"%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n", + "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})", + "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"dd MMM yyyy HH:mm:ss" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "storm_drpc", + "storm_logviewer", + "storm_nimbus", + "storm_supervisor", + "storm_ui", + "storm_worker" + ] + + } + + }, + "log4j_format":"", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss.SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "zookeeper" + ] + + } + + }, + "log4j_format":"%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}-%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\@%{INT:line_number}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "hdfs_audit" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:evtTime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:evtTime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "evtTime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"keyvalue", + "sort_order":1, + "conditions":{ + "fields":{ + "type":[ + "hdfs_audit" + ] + + } + + }, + "source_field":"log_message", + "value_split":"=", + "field_split":"\t", + "post_map_values":{ + "src":{ + "map_fieldname":{ + "new_fieldname":"resource" + } + + }, + "ip":{ + "map_fieldname":{ + "new_fieldname":"cliIP" + } + + }, + "allowed":[ + { + "map_fieldvalue":{ + "pre_value":"true", + "post_value":"1" + } + + }, + { + "map_fieldvalue":{ + "pre_value":"false", + "post_value":"0" + } + + }, + { + "map_fieldname":{ + "new_fieldname":"result" + } + + } + + ], + "cmd":{ + "map_fieldname":{ + "new_fieldname":"action" + } + + }, + "proto":{ + "map_fieldname":{ + "new_fieldname":"cliType" + } + + }, + "callerContext":{ + "map_fieldname":{ + "new_fieldname":"req_caller_id" + } + + } + + } + + }, + { + "filter":"grok", + "sort_order":2, + "source_field":"ugi", + "remove_source_field":"false", + "conditions":{ + "fields":{ + "type":[ + "hdfs_audit" + ] + + } + + }, + "message_pattern":"%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}", + "post_map_values":{ + "user":{ + "map_fieldname":{ + "new_fieldname":"reqUser" + } + + }, + "x_user":{ + "map_fieldname":{ + "new_fieldname":"reqUser" + } + + }, + "p_user":{ + "map_fieldname":{ + "new_fieldname":"reqUser" + } + + }, + "k_user":{ + "map_fieldname":{ + "new_fieldname":"proxyUsers" + } + + }, + "p_authType":{ + "map_fieldname":{ + "new_fieldname":"authType" + } + + }, + "k_authType":{ + "map_fieldname":{ + "new_fieldname":"proxyAuthType" + } + + } + + } + + } + + ], + "output":[ + { + "is_enabled":"{{solr_service_logs_enable}}", + "comment":"Output to solr for service logs", + "destination":"solr", + "zk_hosts":"{{zookeeper_quorum}}{{solr_znode}}", + "collection":"{{solr_collection_service_logs}}", + "number_of_shards": "{{logsearch_numshards}}", + "splits_interval_mins": "{{service_logs_collection_splits_interval_mins}}", + "conditions":{ + "fields":{ + "rowtype":[ + "service" + ] + + } + + } + + }, + { + "comment":"Output to solr for audit records", + "is_enabled":"{{solr_audit_logs_enable}}", + "destination":"solr", + "zk_hosts":"{{zookeeper_quorum}}{{solr_znode}}", + "collection":"{{solr_collection_audit_logs}}", + "number_of_shards": "{{logsearch_numshards}}", + "splits_interval_mins": "{{audit_logs_collection_splits_interval_mins}}", + "conditions":{ + "fields":{ + "rowtype":[ + "audit" + ] + + } + + } + + }, + { + "is_enabled":"{{kafka_service_logs_enable}}", + "destination":"kafka", + "broker_list":"{{kafka_broker_list}}", + "topic":"{{kafka_topic_service_logs}}", + "kafka.security.protocol":"{{kafka_security_protocol}}", + "kafka.sasl.kerberos.service.name":"{{kafka_kerberos_service_name}}", + "conditions":{ + "fields":{ + "rowtype":[ + "service" + ] + + } + + } + + }, + { + "is_enabled":"{{kafka_topic_service_logs}}", + "destination":"kafka", + "broker_list":"{{kafka_broker_list}}", + "topic":"{{kafka_topic_audit_logs}}", + "kafka.security.protocol":"{{kafka_security_protocol}}", + "kafka.sasl.kerberos.service.name":"{{kafka_kerberos_service_name}}", + "conditions":{ + "fields":{ + "rowtype":[ + "audit" + ] + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/filters.config.json ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/filters.config.json b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/filters.config.json new file mode 100644 index 0000000..9493c6c --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/filters.config.json @@ -0,0 +1,626 @@ +{ + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "accumulo_master" + ] + + } + + }, + "log4j_format":"%d{ISO8601} [%-8c{2}] %-5p: %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "comment":"This one has one extra space after LEVEL", + "conditions":{ + "fields":{ + "type":[ + "accumulo_gc", + "accumulo_monitor", + "accumulo_tracer", + "accumulo_tserver" + ] + + } + + }, + "log4j_format":"%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "atlas_app", + "falcon_app" + ] + + } + + }, + "log4j_format":"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{SPACE}-%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}~%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ams_collector" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %p %c: %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ams_hbase_master", + "ams_hbase_regionserver", + "hbase_master", + "hbase_regionserver" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_agent" + ] + + } + + }, + "log4j_format":"", + "multiline_pattern":"^(%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime} %{JAVAFILE:file}:%{INT:line_number} - %{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + }, + "level":{ + "map_fieldvalue":{ + "pre_value":"WARNING", + "post_value":"WARN" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_server" + ] + + } + + }, + "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n", + "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})", + "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"dd MMM yyyy HH:mm:ss" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "hdfs_datanode", + "hdfs_journalnode", + "hdfs_secondarynamenode", + "hdfs_namenode", + "hdfs_zkfc", + "knox_gateway", + "knox_cli", + "knox_ldap", + "mapred_historyserver", + "yarn_historyserver", + "yarn_jobsummary", + "yarn_nodemanager", + "yarn_resourcemanager", + "yarn_timelineserver" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "hive_hiveserver2", + "hive_metastore" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]:%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "kafka_controller", + "kafka_request", + "kafka_logcleaner" + ] + + } + + }, + "log4j_format":"[%d] %p %m (%c)%n", + "multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])", + "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "comment":"Suppose to be same log4j pattern as other kafka processes, but some reason thread is not printed", + "conditions":{ + "fields":{ + "type":[ + "kafka_server", + "kafka_statechange" + ] + + } + + }, + "log4j_format":"[%d] %p %m (%c)%n", + "multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])", + "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "oozie_app" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %5p %c{1}:%L - SERVER[${oozie.instance.id}] %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{DATA:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "logsearch_app", + "logsearch_feeder", + "logsearch_perf", + "ranger_admin", + "ranger_dbpatch" + ] + + } + + }, + "log4j_format":"%d [%t] %-5p %C{6} (%F:%L) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ranger_kms" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p %c{1} - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ranger_usersync" + ] + + } + + }, + "log4j_format":"%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n", + "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})", + "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"dd MMM yyyy HH:mm:ss" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "storm_drpc", + "storm_logviewer", + "storm_nimbus", + "storm_supervisor", + "storm_ui", + "storm_worker" + ] + + } + + }, + "log4j_format":"", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss.SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "zookeeper" + ] + + } + + }, + "log4j_format":"%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}-%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\@%{INT:line_number}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "hdfs_audit" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:evtTime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:evtTime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "evtTime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"keyvalue", + "sort_order":1, + "conditions":{ + "fields":{ + "type":[ + "hdfs_audit" + ] + + } + + }, + "source_field":"log_message", + "value_split":"=", + "field_split":"\t", + "post_map_values":{ + "src":{ + "map_fieldname":{ + "new_fieldname":"resource" + } + + }, + "ip":{ + "map_fieldname":{ + "new_fieldname":"cliIP" + } + + }, + "allowed":[ + { + "map_fieldvalue":{ + "pre_value":"true", + "post_value":"1" + } + + }, + { + "map_fieldvalue":{ + "pre_value":"false", + "post_value":"0" + } + + }, + { + "map_fieldname":{ + "new_fieldname":"result" + } + + } + + ], + "cmd":{ + "map_fieldname":{ + "new_fieldname":"action" + } + + }, + "proto":{ + "map_fieldname":{ + "new_fieldname":"cliType" + } + + }, + "callerContext":{ + "map_fieldname":{ + "new_fieldname":"req_caller_id" + } + + } + + } + + }, + { + "filter":"grok", + "sort_order":2, + "source_field":"ugi", + "remove_source_field":"false", + "conditions":{ + "fields":{ + "type":[ + "hdfs_audit" + ] + + } + + }, + "message_pattern":"%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}", + "post_map_values":{ + "user":{ + "map_fieldname":{ + "new_fieldname":"reqUser" + } + + }, + "x_user":{ + "map_fieldname":{ + "new_fieldname":"reqUser" + } + + }, + "p_user":{ + "map_fieldname":{ + "new_fieldname":"reqUser" + } + + }, + "k_user":{ + "map_fieldname":{ + "new_fieldname":"proxyUsers" + } + + }, + "p_authType":{ + "map_fieldname":{ + "new_fieldname":"authType" + } + + }, + "k_authType":{ + "map_fieldname":{ + "new_fieldname":"proxyAuthType" + } + + } + + } + + } + + ] +} http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/global.config.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/global.config.json.j2 b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/global.config.json.j2 new file mode 100644 index 0000000..cd51118 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/global.config.json.j2 @@ -0,0 +1,28 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "global":{ + "add_fields":{ + "cluster":"{{cluster_name}}" + }, + "source":"file", + "tail":"true", + "gen_event_md5":"true", + "start_position":"beginning" + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/grok-patterns ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/grok-patterns b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/grok-patterns new file mode 100644 index 0000000..d25a78b --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/grok-patterns @@ -0,0 +1,145 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#Updated JAVACLASS to be same as JAVAFILE. Because if class doesn't have package, then it doesn't work. +JAVACLASS (?:[A-Za-z$0-9_. -]+) +#JAVACLASS (?:[a-zA-Z$_][a-zA-Z$_0-9]*\.)*[a-zA-Z$_][a-zA-Z$_0-9]* +#JAVACLASS (?:[a-zA-Z0-9-]+\.)+[A-Za-z0-9$]+ + +#Space is an allowed character to match special cases like 'Native Method' or 'Unknown Source' +JAVAFILE (?:[A-Za-z0-9_. -]+) +#Allow special <init> or <clinit> method +JAVAMETHOD (?:(<init>)|(<clinit>)|[a-zA-Z$_][a-zA-Z$_0-9]*) +#Line number is optional in special cases 'Native method' or 'Unknown source' +JAVASTACKTRACEPART %{SPACE}at %{JAVACLASS:class}\.%{JAVAMETHOD:method}\(%{JAVAFILE:file}(?::%{NUMBER:line})?\) +# Java Logs +JAVATHREAD (?:[A-Z]{2}-Processor[\d]+) + +JAVASTACKTRACEPART at %{JAVACLASS:class}\.%{WORD:method}\(%{JAVAFILE:file}:%{NUMBER:line}\) +JAVALOGMESSAGE (.*) +# MMM dd, yyyy HH:mm:ss eg: Jan 9, 2014 7:13:13 AM +CATALINA_DATESTAMP %{MONTH} %{MONTHDAY}, 20%{YEAR} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) (?:AM|PM) +# yyyy-MM-dd HH:mm:ss,SSS ZZZ eg: 2014-01-09 17:32:25,527 -0800 +TOMCAT_DATESTAMP 20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) %{ISO8601_TIMEZONE} +CATALINALOG %{CATALINA_DATESTAMP:timestamp} %{JAVACLASS:class} %{JAVALOGMESSAGE:logmessage} +# 2014-01-09 20:03:28,269 -0800 | ERROR | com.example.service.ExampleService - something compeletely unexpected happened... +TOMCATLOG %{TOMCAT_DATESTAMP:timestamp} \| %{LOGLEVEL:level} \| %{JAVACLASS:class} - %{JAVALOGMESSAGE:logmessage} + +USERNAME [a-zA-Z0-9._-]+ +USER %{USERNAME} +EMAILLOCALPART [a-zA-Z][a-zA-Z0-9_.+-=:]+ +EMAILADDRESS %{EMAILLOCALPART}@%{HOSTNAME} +HTTPDUSER %{EMAILADDRESS}|%{USER} +INT (?:[+-]?(?:[0-9]+)) +BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+))) +NUMBER (?:%{BASE10NUM}) +BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+)) +BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b + +POSINT \b(?:[1-9][0-9]*)\b +NONNEGINT \b(?:[0-9]+)\b +WORD \b\w+\b +NOTSPACE \S+ +SPACE \s* +DATA .*? +GREEDYDATA .* +QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``)) +UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12} + +# Networking +MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC}) +CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4}) +WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2}) +COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2}) +IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5 ]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? +IPV4 (?<![0-9])(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))(?![0-9]) +IP (?:%{IPV6}|%{IPV4}) +HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b) +IPORHOST (?:%{IP}|%{HOSTNAME}) +HOSTPORT %{IPORHOST}:%{POSINT} + +# paths +PATH (?:%{UNIXPATH}|%{WINPATH}) +UNIXPATH (/([\w_%!$@:.,~-]+|\\.)*)+ +TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+)) +WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+ +URIPROTO [A-Za-z]+(\+[A-Za-z+]+)? +URIHOST %{IPORHOST}(?::%{POSINT:port})? +# uripath comes loosely from RFC1738, but mostly from what Firefox +# doesn't turn into %XX +URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+ +#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)? +URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]<>]* +URIPATHPARAM %{URIPATH}(?:%{URIPARAM})? +URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})? + +# Months: January, Feb, 3, 03, 12, December +MONTH \b(?:Jan(?:uary|uar)?|Feb(?:ruary|ruar)?|M(?:a|ä)?r(?:ch|z)?|Apr(?:il)?|Ma(?:y|i)?|Jun(?:e|i)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|O(?:c|k)?t(?:ober)?|Nov(?:ember)?|De(?:c|z)(?:ember)?)\b +MONTHNUM (?:0?[1-9]|1[0-2]) +MONTHNUM2 (?:0[1-9]|1[0-2]) +MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) + +# Days: Monday, Tue, Thu, etc... +DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?) + +# Years? +YEAR (?>\d\d){1,2} +HOUR (?:2[0123]|[01]?[0-9]) +MINUTE (?:[0-5][0-9]) +# '60' is a leap second in most time standards and thus is valid. +SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?) +TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9]) +# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it) +DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR} +DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR} +ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE})) +ISO8601_SECOND (?:%{SECOND}|60) +TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}? +DATE %{DATE_US}|%{DATE_EU} +DATESTAMP %{DATE}[- ]%{TIME} +TZ (?:[PMCE][SD]T|UTC) +DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ} +DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE} +DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR} +DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND} +HTTPDERROR_DATE %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR} + +# Syslog Dates: Month Day HH:MM:SS +SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME} +PROG [\x21-\x5a\x5c\x5e-\x7e]+ +SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])? +SYSLOGHOST %{IPORHOST} +SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}> +HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT} + +# Shortcuts +QS %{QUOTEDSTRING} + +# Log formats +SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: +COMMONAPACHELOG %{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) +COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent} +HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:errormsg} +HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message} +HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG} + + +# Log Levels +LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?) + + +# Custom +USER_SYNC_DATE %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/input.config.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/input.config.json.j2 b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/input.config.json.j2 new file mode 100644 index 0000000..bc48503 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/input.config.json.j2 @@ -0,0 +1,284 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"accumulo_gc", + "rowtype":"service", + "path":"{{accumulo_log_dir}}/gc_*.log" + }, + { + "type":"accumulo_master", + "rowtype":"service", + "path":"{{accumulo_log_dir}}/master_*.log" + }, + { + "type":"accumulo_monitor", + "rowtype":"service", + "path":"{{accumulo_log_dir}}/monitor_*.log" + }, + { + "type":"accumulo_tracer", + "rowtype":"service", + "path":"{{accumulo_log_dir}}/tracer_*.log" + }, + { + "type":"accumulo_tserver", + "rowtype":"service", + "path":"{{accumulo_log_dir}}/tserver_*.log" + }, + { + "type":"atlas_app", + "rowtype":"service", + "path":"{{atlas_log_dir}}/application.log" + }, + { + "type":"ambari_agent", + "rowtype":"service", + "path":"{{ambari_agent_log_dir}}/ambari-agent.log" + }, + { + "type":"ambari_server", + "rowtype":"service", + "path":"{{ambari_server_log_dir}}/ambari-server.log" + }, + { + "type":"ams_hbase_master", + "rowtype":"service", + "path":"{{metrics_collector_log_dir}}/hbase-ams-master-*.log" + }, + { + "type":"ams_hbase_regionserver", + "rowtype":"service", + "path":"{{metrics_collector_log_dir}}/hbase-ams-regionserver-*.log" + }, + { + "type":"ams_collector", + "rowtype":"service", + "path":"{{metrics_collector_log_dir}}/ambari-metrics-collector.log" + }, + { + "type":"falcon_app", + "rowtype":"service", + "path":"{{falcon_log_dir}}/falcon.application.log" + }, + { + "type":"hbase_master", + "rowtype":"service", + "path":"{{hbase_log_dir}}/hbase-hbase-master-*.log" + }, + { + "type":"hbase_regionserver", + "rowtype":"service", + "path":"{{hbase_log_dir}}/hbase-hbase-regionserver-*.log" + }, + { + "type":"hdfs_datanode", + "rowtype":"service", + "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-datanode-*.log" + }, + { + "type":"hdfs_namenode", + "rowtype":"service", + "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-namenode-*.log" + }, + { + "type":"hdfs_journalnode", + "rowtype":"service", + "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-journalnode-*.log" + }, + { + "type":"hdfs_secondarynamenode", + "rowtype":"service", + "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-secondarynamenode-*.log" + }, + { + "type":"hdfs_zkfc", + "rowtype":"service", + "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-zkfc-*.log" + }, + { + "type":"hive_hiveserver2", + "rowtype":"service", + "path":"{{hive_log_dir}}/hiveserver2.log" + }, + { + "type":"hive_metastore", + "rowtype":"service", + "path":"{{hive_log_dir}}/hivemetastore.log" + }, + { + "type":"kafka_controller", + "rowtype":"service", + "path":"{{kafka_log_dir}}/controller.log" + }, + { + "type":"kafka_request", + "rowtype":"service", + "path":"{{kafka_log_dir}}/kafka-request.log" + }, + { + "type":"kafka_logcleaner", + "rowtype":"service", + "path":"{{kafka_log_dir}}/log-cleaner.log" + }, + { + "type":"kafka_server", + "rowtype":"service", + "path":"{{kafka_log_dir}}/server.log" + }, + { + "type":"kafka_statechange", + "rowtype":"service", + "path":"{{kafka_log_dir}}/state-change.log" + }, + { + "type":"knox_gateway", + "rowtype":"service", + "path":"{{knox_log_dir}}/gateway.log" + }, + { + "type":"knox_cli", + "rowtype":"service", + "path":"{{knox_log_dir}}/knoxcli.log" + }, + { + "type":"knox_ldap", + "rowtype":"service", + "path":"{{knox_log_dir}}/ldap.log" + }, + { + "type":"mapred_historyserver", + "rowtype":"service", + "path":"{{mapred_log_dir_prefix}}/mapred/mapred-mapred-historyserver*.log" + }, + { + "type":"logsearch_app", + "rowtype":"service", + "path":"{{logsearch_log_dir}}/logsearch.log" + }, + { + "type":"logsearch_feeder", + "rowtype":"service", + "path":"{{logfeeder_log_dir}}/logfeeder.log" + }, + { + "type":"logsearch_perf", + "rowtype":"service", + "path":"{{logsearch_log_dir}}/logsearch-performance.log" + }, + { + "type":"ranger_admin", + "rowtype":"service", + "path":"{{ranger_admin_log_dir}}/xa_portal.log" + }, + { + "type":"ranger_dbpatch", + "is_enabled":"true", + "path":"{{ranger_admin_log_dir}}/ranger_db_patch.log" + }, + { + "type":"ranger_kms", + "rowtype":"service", + "path":"{{ranger_kms_log_dir}}/kms.log" + }, + { + "type":"ranger_usersync", + "rowtype":"service", + "path":"{{ranger_usersync_log_dir}}/usersync.log" + }, + { + "type":"oozie_app", + "rowtype":"service", + "path":"{{oozie_log_dir}}/oozie.log" + }, + { + "type":"yarn_nodemanager", + "rowtype":"service", + "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-nodemanager-*.log" + }, + { + "type":"yarn_resourcemanager", + "rowtype":"service", + "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-resourcemanager-*.log" + }, + { + "type":"yarn_timelineserver", + "rowtype":"service", + "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-timelineserver-*.log" + }, + { + "type":"yarn_historyserver", + "rowtype":"service", + "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-historyserver-*.log" + }, + { + "type":"yarn_jobsummary", + "rowtype":"service", + "path":"{{yarn_log_dir_prefix}}/yarn/hadoop-mapreduce.jobsummary.log" + }, + { + "type":"storm_drpc", + "rowtype":"service", + "path":"{{storm_log_dir}}/drpc.log" + }, + { + "type":"storm_logviewer", + "rowtype":"service", + "path":"{{storm_log_dir}}/logviewer.log" + }, + { + "type":"storm_nimbus", + "rowtype":"service", + "path":"{{storm_log_dir}}/nimbus.log" + }, + { + "type":"storm_supervisor", + "rowtype":"service", + "path":"{{storm_log_dir}}/supervisor.log" + }, + { + "type":"storm_ui", + "rowtype":"service", + "path":"{{storm_log_dir}}/ui.log" + }, + { + "type":"storm_worker", + "rowtype":"service", + "path":"{{storm_log_dir}}/*worker*.log" + }, + { + "type":"zookeeper", + "rowtype":"service", + "path":"{{zk_log_dir}}/zookeeper/zookeeper*.out" + }, + { + "type":"hdfs_audit", + "rowtype":"audit", + "is_enabled":"true", + "add_fields":{ + "logType":"HDFSAudit", + "enforcer":"hadoop-acl", + "repoType":"1", + "repo":"hdfs" + }, + "path":"{{hdfs_log_dir_prefix}}/hdfs/hdfs-audit.log" + } + + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml new file mode 100644 index 0000000..0717477 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml @@ -0,0 +1,62 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd"> + +<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/";> + <appender name="console" class="org.apache.log4j.ConsoleAppender"> + <param name="Target" value="System.out" /> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n" /> + <!-- <param name="ConversionPattern" value="%d [%t] %-5p %c %x - %m%n"/> --> + </layout> + </appender> + + <appender name="daily_rolling_file" class="org.apache.log4j.DailyRollingFileAppender"> + <param name="file" value="logs/logsearch-logfeeder.log" /> + <param name="datePattern" value="'.'yyyy-MM-dd" /> + <param name="append" value="true" /> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n"/> + </layout> + </appender> + + <!-- Logs to suppress BEGIN --> + <category name="org.apache.solr.common.cloud.ZkStateReader" additivity="false"> + <priority value="error" /> + <appender-ref ref="rolling_file" /> + </category> + + <category name="apache.solr.client.solrj.impl.CloudSolrClient" additivity="false"> + <priority value="fatal" /> + <appender-ref ref="rolling_file" /> + </category> + <!-- Logs to suppress END --> + + <category name="org.apache.ambari.logfeeder" additivity="false"> + <priority value="info" /> + <appender-ref ref="console" /> + <!-- <appender-ref ref="daily_rolling_file" /> --> + </category> + + <root> + <priority value="warn" /> + <!-- <appender-ref ref="console" /> --> + <!-- <appender-ref ref="daily_rolling_file" /> --> + </root> + +</log4j:configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml.j2 ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml.j2 b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml.j2 new file mode 100644 index 0000000..4338ee3 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml.j2 @@ -0,0 +1,60 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd"> + +<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/";> + <appender name="console" class="org.apache.log4j.ConsoleAppender"> + <param name="Target" value="System.out" /> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n" /> + <!-- <param name="ConversionPattern" value="%d [%t] %-5p %c %x - %m%n"/> --> + </layout> + </appender> + + <appender name="rolling_file" class="org.apache.log4j.RollingFileAppender"> + <param name="file" value="{{logfeeder_log_dir}}/logfeeder.log" /> + <param name="append" value="true" /> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n"/> + </layout> + </appender> + + <!-- Logs to suppress BEGIN --> + <category name="org.apache.solr.common.cloud.ZkStateReader" additivity="false"> + <priority value="error" /> + <appender-ref ref="rolling_file" /> + </category> + + <category name="apache.solr.client.solrj.impl.CloudSolrClient" additivity="false"> + <priority value="fatal" /> + <appender-ref ref="rolling_file" /> + </category> + + <!-- Logs to suppress END --> + + <category name="org.apache.ambari.logfeeder" additivity="false"> + <priority value="{{logfeeder_log_level}}" /> + <appender-ref ref="rolling_file" /> + </category> + + <root> + <level value="warn" /> + <!-- <appender-ref ref="console" /> --> + <appender-ref ref="rolling_file" /> + </root> +</log4j:configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties new file mode 100644 index 0000000..22f3b78 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +logfeeder.checkpoint.folder= +metrics.collector.hosts= + +#filter config +logfeeder.solr.url= +logfeeder.solr.zkhosts= +logfeeder.solr.core.history=history +logfeeder.log.filter.enable=true +#Internal to fetch filter config from solr in sec +logfeeder.solr.config.internal=5 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/output.config.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/output.config.json.j2 b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/output.config.json.j2 new file mode 100644 index 0000000..d0aea47 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/output.config.json.j2 @@ -0,0 +1,97 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "output":[ + { + "is_enabled":"{{solr_service_logs_enable}}", + "comment":"Output to solr for service logs", + "destination":"solr", + "zk_hosts":"{{zookeeper_quorum}}{{solr_znode}}", + "collection":"{{solr_collection_service_logs}}", + "number_of_shards": "{{logsearch_numshards}}", + "splits_interval_mins": "{{service_logs_collection_splits_interval_mins}}", + "conditions":{ + "fields":{ + "rowtype":[ + "service" + ] + + } + + } + + }, + { + "comment":"Output to solr for audit records", + "is_enabled":"{{solr_audit_logs_enable}}", + "destination":"solr", + "zk_hosts":"{{zookeeper_quorum}}{{solr_znode}}", + "collection":"{{solr_collection_audit_logs}}", + "number_of_shards": "{{logsearch_numshards}}", + "splits_interval_mins": "{{audit_logs_collection_splits_interval_mins}}", + "conditions":{ + "fields":{ + "rowtype":[ + "audit" + ] + + } + + } + + }, + { + "is_enabled":"{{kafka_service_logs_enable}}", + "destination":"kafka", + "broker_list":"{{kafka_broker_list}}", + "topic":"{{kafka_topic_service_logs}}", + "kafka.security.protocol":"{{kafka_security_protocol}}", + "kafka.sasl.kerberos.service.name":"{{kafka_kerberos_service_name}}", + "conditions":{ + "fields":{ + "rowtype":[ + "service" + ] + + } + + } + + }, + { + "is_enabled":"{{kafka_topic_service_logs}}", + "destination":"kafka", + "broker_list":"{{kafka_broker_list}}", + "topic":"{{kafka_topic_audit_logs}}", + "kafka.security.protocol":"{{kafka_security_protocol}}", + "kafka.sasl.kerberos.service.name":"{{kafka_kerberos_service_name}}", + "conditions":{ + "fields":{ + "rowtype":[ + "audit" + ] + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh new file mode 100644 index 0000000..1e7185a --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +curr_dir=`pwd` +cd `dirname $0`; script_dir=`pwd`; cd $curr_dir + +foreground=0 +if [ "$1" = "-foreground" ]; then + foreground=1 + shift +fi + +if [ ! -z "$LOGFEEDER_INCLUDE" ]; then + source $LOGFEEDER_INCLUDE +fi + +JAVA=java +if [ -x $JAVA_HOME/bin/java ]; then + JAVA=$JAVA_HOME/bin/java +fi + +if [ "$LOGFEEDER_JAVA_MEM" = "" ]; then + LOGFEEDER_JAVA_MEM="-Xmx512m" +fi + +if [ "$LOGFILE" = "" ]; then + LOGFILE="/var/log/logfeeder/logfeeder.out" +fi + +if [ "$PID_FILE" = "" ]; then + LOGFEEDER_PID_DIR=$HOME + PID_FILE=$LOGFEEDER_PID_DIR/logsearch-logfeeder-$USER.pid +fi + +if [ "$LOGFEEDER_CONF_DIR" = "" ]; then + LOGFEEDER_CONF_DIR="/etc/logfeeder/conf" +fi + +LOGFEEDER_GC_LOGFILE=`dirname $LOGFILE`/logfeeder_gc.log +LOGFEEDER_GC_OPTS="-XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$LOGFEEDER_GC_LOGFILE" + +#LOGFEEDER_JAVA_OPTS= +#JMX="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=2098" + +if [ $foreground -eq 0 ]; then + if [ -f ${PID_FILE} ]; then + PID=`cat ${PID_FILE}` + if kill -0 $PID 2>/dev/null; then + echo "logfeeder already running (${PID}) killing..." + kill $PID 2>/dev/null + sleep 5 + if kill -0 $PID 2>/dev/null; then + echo "logfeeder still running. Will kill process forcefully in another 10 seconds..." + sleep 10 + kill -9 $PID 2>/dev/null + sleep 2 + fi + fi + + if kill -0 $PID 2>/dev/null; then + echo "ERROR: Even after all efforts to stop logfeeder, it is still running. pid=$PID. Please manually kill the service and try again." + exit 1 + fi + fi + + echo "Starting logfeeder. Output file=$LOGFILE pid_file=$PID_FILE" + #LOGFEEDER_CLI_CLASSPATH= + #set -x + nohup $JAVA -cp "$LOGFEEDER_CLI_CLASSPATH:$LOGFEEDER_CONF_DIR:$script_dir/libs/*:$script_dir/classes:$script_dir/LogProcessor.jar" $LOGFEEDER_GC_OPTS $LOGFEEDER_JAVA_MEM $LOGFEEDER_JAVA_OPTS $JMX org.apache.ambari.logfeeder.LogFeeder $* > $LOGFILE 2>&1 & + echo $! > $PID_FILE +else + $JAVA -cp "$LOGFEEDER_CLI_CLASSPATH:$LOGFEEDER_CONF_DIR:$script_dir/libs/*:$script_dir/classes:$script_dir/LogProcessor.jar" $LOGFEEDER_JAVA_MEM $LOGFEEDER_JAVA_OPTS $JMX org.apache.ambari.logfeeder.LogFeeder $* +fi +
