AMBARI-22364. Log Feeder should be able to run locally with dockerized dev env by default (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/654404dc Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/654404dc Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/654404dc Branch: refs/heads/branch-feature-AMBARI-21674 Commit: 654404dc8c545f47bafd8a446b99ecb251d83982 Parents: 063ba36 Author: Oliver Szabo <[email protected]> Authored: Tue Nov 7 14:22:38 2017 +0100 Committer: Oliver Szabo <[email protected]> Committed: Tue Nov 7 17:08:04 2017 +0100 ---------------------------------------------------------------------- .../ambari-logsearch-logfeeder/README.md | 17 +++++----- .../ambari-logsearch-logfeeder/pom.xml | 4 ++- .../src/main/resources/log-samples/.gitignore | 4 +++ .../log-samples/logs/service_sample.txt | 3 ++ .../log-samples/shipper-conf/global.config.json | 10 ++++++ .../shipper-conf/input.config-sample.json | 31 ++++++++++++++++++ .../shipper-conf/output.config-sample.json | 34 ++++++++++++++++++++ .../src/main/resources/log4j.xml | 4 +-- .../src/main/resources/logfeeder.properties | 31 ++++++++++-------- 9 files changed, 113 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/654404dc/ambari-logsearch/ambari-logsearch-logfeeder/README.md ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/README.md b/ambari-logsearch/ambari-logsearch-logfeeder/README.md index 5a64daf..d2e55f0 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/README.md +++ b/ambari-logsearch/ambari-logsearch-logfeeder/README.md @@ -22,19 +22,18 @@ limitations under the License. Log Feeder is a component of the Log Search service that reads logs, parses them and stores them in Apache Solr for the purpose of later analysis. -# Compilation +## Start locally from maven / IDE + +First you need to start every required service (except logfeeder), go to `ambari-logsearch/docker` folder and run: ```bash -mvn clean compile package +docker-compose up -d zookeeper solr logsearch ``` -# Deploy -## Copy to remote -copy target/logsearch-logfeeder.tgz to host machine -## Setup environment +Secondly, if you are planning to run Log Feeder from an IDE, for running the LogFeeder main methoud, you will need to set the working directory to `ambari/ambari-logsearch/ambari-logsearch-logfeeder` and use `--monitor` as a command line argument. +With Maven, you won't need these steps, just run this command from the ambari-logsearch-logfeeder folder: + ```bash -mkdir /opt/logfeeder -cd /opt/logfeeder -tar xfz ~/logsearch-logfeeder.tar.gz +mvn clean package -DskipTests exec:java ``` # Input Configuration http://git-wip-us.apache.org/repos/asf/ambari/blob/654404dc/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml index b1b6ece..6b7d94f 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml +++ b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml @@ -220,7 +220,9 @@ </executions> <configuration> <mainClass>org.apache.ambari.logfeeder.LogFeeder</mainClass> - <!-- <arguments> <argument></argument> </arguments> --> + <arguments> + <argument>--monitor</argument> + </arguments> </configuration> </plugin> <!-- copy-dependencies --> http://git-wip-us.apache.org/repos/asf/ambari/blob/654404dc/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/.gitignore ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/.gitignore b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/.gitignore new file mode 100644 index 0000000..dfb10d6 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/.gitignore @@ -0,0 +1,4 @@ +logs/*.log +shipper-conf/input.config-*.json +!shipper-conf/input.config-sample.json + http://git-wip-us.apache.org/repos/asf/ambari/blob/654404dc/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/logs/service_sample.txt ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/logs/service_sample.txt b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/logs/service_sample.txt new file mode 100644 index 0000000..21048ac --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/logs/service_sample.txt @@ -0,0 +1,3 @@ +2016-07-13 10:45:49,640 [WARN] Sample log line 1 - warn level +2016-07-13 10:45:49,640 [ERROR] Sample log line 2 - error level +2016-07-13 10:45:50,351 [INFO] Sample log line 3 - info level \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/654404dc/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/global.config.json ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/global.config.json b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/global.config.json new file mode 100644 index 0000000..6b8602c --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/global.config.json @@ -0,0 +1,10 @@ +{ + "global":{ + "add_fields":{ + "cluster":"cl1" + }, + "source":"file", + "tail":"true", + "gen_event_md5":"true" + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/654404dc/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/input.config-sample.json ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/input.config-sample.json b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/input.config-sample.json new file mode 100644 index 0000000..4ab2eb2 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/input.config-sample.json @@ -0,0 +1,31 @@ +{ + "input": [ + { + "type": "service_sample", + "rowtype": "service", + "path": "target/classes/log-samples/logs/service_sample.txt" + } + ], + "filter": [ + { + "filter": "grok", + "conditions": { + "fields": { + "type": [ + "service_sample" + ] + } + }, + "log4j_format": "", + "multiline_pattern": "^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern": "(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values": { + "logtime": { + "map_date": { + "date_pattern": "yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] +} http://git-wip-us.apache.org/repos/asf/ambari/blob/654404dc/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/output.config-sample.json ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/output.config-sample.json b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/output.config-sample.json new file mode 100644 index 0000000..94b44da --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/output.config-sample.json @@ -0,0 +1,34 @@ +{ + "output": [ + { + "is_enabled": "true", + "comment": "Output to solr for service logs", + "destination": "solr", + "zk_connect_string": "localhost:2181", + "type": "service", + "skip_logtime": "true", + "conditions": { + "fields": { + "rowtype": [ + "service" + ] + } + } + }, + { + "comment": "Output to solr for audit records", + "is_enabled": "true", + "destination": "solr", + "zk_connect_string": "localhost:2181", + "type": "audit", + "skip_logtime": "true", + "conditions": { + "fields": { + "rowtype": [ + "audit" + ] + } + } + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/654404dc/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml index 8d5174e..eb20665 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml @@ -28,7 +28,7 @@ </appender> <appender name="daily_rolling_file" class="org.apache.log4j.DailyRollingFileAppender"> - <param name="file" value="logs/logsearch-logfeeder.log" /> + <param name="file" value="target/logs/logsearch-logfeeder.log" /> <param name="datePattern" value="'.'yyyy-MM-dd" /> <param name="append" value="true" /> <layout class="org.apache.log4j.PatternLayout"> @@ -38,7 +38,7 @@ <appender name="rolling_file_json" class="org.apache.ambari.logsearch.appender.LogsearchRollingFileAppender"> - <param name="file" value="logs/logsearch-logfeeder.json" /> + <param name="file" value="target/logs/logsearch-logfeeder.json" /> <param name="append" value="true" /> <param name="maxFileSize" value="10MB" /> <param name="maxBackupIndex" value="10" /> http://git-wip-us.apache.org/repos/asf/ambari/blob/654404dc/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties index 74ea0ef..115778b 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties @@ -13,19 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -logfeeder.checkpoint.folder= +cluster.name=cl1 +logfeeder.checkpoint.folder=target/checkpoints logfeeder.metrics.collector.hosts= - -#filter config -logfeeder.log.filter.enable=false +logfeeder.config.dir=target/classes/log-samples/shipper-conf/ +logfeeder.config.files=target/classes/log-samples/shipper-conf/global.config.json,\ + target/classes/log-samples/shipper-conf/input.config-sample.json,\ + target/classes/log-samples/shipper-conf/output.config-sample.json +logfeeder.log.filter.enable=true logfeeder.solr.config.interval=5 -logfeeder.solr.zk_connect_string= -logfeeder.solr.url= - -logfeeder.solr.kerberos.enable=false -logfeeder.solr.jaas.file=/usr/lib/ambari-logsearch-logfeeder/logsearch_solr_jaas.conf - -#logfeeder tmp dir -logfeeder.tmp.dir=/tmp/$username/logfeeder/ - logfeeder.solr.core.config.name=history +logfeeder.solr.zk_connect_string=localhost:2181 +logfeeder.cache.enabled=true +logfeeder.cache.size=100 +logfeeder.cache.key.field=log_message +logfeeder.cache.dedup.interval=1000 +logfeeder.cache.last.dedup.enabled=true +logsearch.config.zk_connect_string=localhost:2181 +logfeeder.include.default.level=FATAL,ERROR,WARN,INFO,DEBUG,TRACE,UNKNOWN + +#logfeeder tmp dir +logfeeder.tmp.dir=target/tmp
