Gehel has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/371950 )
Change subject: [WIP] Send logs to logstash
......................................................................
[WIP] Send logs to logstash
Adds the logstash-logback-encoder as a dependency and configure it to send
logs to logstash.
The LogstashTcpSocketAppender is used as it seems to be a good way to ensure
large stacktraces can be sent without issue. It works asynchronously to
ensure that the logging thread isn't blocked. It uses an LMAX Disruptor for
queuing, which should provide more than enoug performance and robustness.
In case of overflow, log messages are dropped to prioritize performances.
Note that minimal testing on wdqs-beta show that messages are sent, but not
parsed by logstash. More investigation needed before merging.
Bug: T172710
Change-Id: Ie9075dfbc34055748976648fa3d06a91fd2d9563
---
M dist/src/script/runBlazegraph.sh
M dist/src/script/runUpdate.sh
M pom.xml
M tools/pom.xml
M tools/src/main/resources/logback.xml
M war/pom.xml
M war/src/main/resources/logback.xml
7 files changed, 43 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/wikidata/query/rdf
refs/changes/50/371950/1
diff --git a/dist/src/script/runBlazegraph.sh b/dist/src/script/runBlazegraph.sh
index 4be9c04..e7cb21e 100755
--- a/dist/src/script/runBlazegraph.sh
+++ b/dist/src/script/runBlazegraph.sh
@@ -9,6 +9,8 @@
PORT=${PORT:-"9999"}
DIR=${DIR:-`dirname $0`}
HEAP_SIZE=${HEAP_SIZE:-"16g"}
+LOGSTASH_SERVER=${LOGSTASH_SERVER:-"localhost"}
+LOGSTASH_PORT=${LOGSTASH_PORT:-"11514"}
MEMORY=${MEMORY:-"-Xms${HEAP_SIZE} -Xmx${HEAP_SIZE}"}
GC_LOGS=${GC_LOGS:-"-Xloggc:/var/log/wdqs/wdqs-blazegraph_jvm_gc.%p.log \
-XX:+PrintGCDetails \
@@ -66,6 +68,8 @@
-Dorg.wikidata.query.rdf.blazegraph.mwapi.MWApiServiceFactory.config=$DIR/mwservices.json
\
-Dcom.bigdata.rdf.sail.webapp.client.HttpClientConfigurator=org.wikidata.query.rdf.blazegraph.ProxiedHttpConnectionFactory
\
-Dhttp.userAgent="${USER_AGENT}" \
+ -Dlogstash.host="${LOGSTASH_SERVER}" \
+ -Dlogstash.port="${LOGSTASH_PORT}" \
${BLAZEGRAPH_OPTS} \
-jar jetty-runner*.jar \
--host $HOST \
diff --git a/dist/src/script/runUpdate.sh b/dist/src/script/runUpdate.sh
index e56a96f..ea4bfbb 100755
--- a/dist/src/script/runUpdate.sh
+++ b/dist/src/script/runUpdate.sh
@@ -1,5 +1,9 @@
#!/usr/bin/env bash
+if [ -r /etc/default/wdqs-updater ]; then
+ . /etc/default/wdqs-updater
+fi
+
HOST=http://localhost:9999
CONTEXT=bigdata
MEMORY="-Xms2g -Xmx2g"
@@ -13,6 +17,8 @@
-XX:+UseGCLogFileRotation \
-XX:NumberOfGCLogFiles=10 \
-XX:GCLogFileSize=20M"
+LOGSTASH_SERVER=${LOGSTASH_SERVER:-"localhost"}
+LOGSTASH_PORT=${LOGSTASH_PORT:-"11514"}
NAMESPACE=wdq
UPDATER_OPTS=${UPDATER_OPTS:-""}
@@ -61,6 +67,7 @@
if [ -f updater-logs.xml ]; then
LOG="-Dlogback.configurationFile=updater-logs.xml"
fi
+LOG="${LOG} -Dlogstash.host=${LOGSTASH_SERVER}
-Dlogstash.port=${LOGSTASH_PORT}"
CP=lib/wikidata-query-tools-*-jar-with-dependencies.jar
MAIN=org.wikidata.query.rdf.tool.Update
diff --git a/pom.xml b/pom.xml
index 866e8a8..4a75363 100644
--- a/pom.xml
+++ b/pom.xml
@@ -555,6 +555,12 @@
<version>1.4.01</version>
</dependency>
<dependency>
+ <groupId>net.logstash.logback</groupId>
+ <artifactId>logstash-logback-encoder</artifactId>
+ <version>4.11</version>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
<groupId>org.jhades</groupId>
<artifactId>jhades</artifactId>
<version>1.0.4</version>
diff --git a/tools/pom.xml b/tools/pom.xml
index faeca64..28d668c 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -115,6 +115,11 @@
<artifactId>common</artifactId>
</dependency>
<dependency>
+ <groupId>net.logstash.logback</groupId>
+ <artifactId>logstash-logback-encoder</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
<scope>runtime</scope>
diff --git a/tools/src/main/resources/logback.xml
b/tools/src/main/resources/logback.xml
index 206a188..da0d47b 100644
--- a/tools/src/main/resources/logback.xml
+++ b/tools/src/main/resources/logback.xml
@@ -7,9 +7,17 @@
</encoder>
</appender>
+ <appender name="logstash"
class="net.logstash.logback.appender.LogstashTcpSocketAppender">
+
<destination>${logstash.host:-localhost}:${logstash.port:-11514}</destination>
+ <encoder class="net.logstash.logback.encoder.LogstashEncoder">
+ <customFields>{"type":"wdqs-updater"}</customFields>
+ </encoder>
+ </appender>
+
<logger name="org.wikidata.query.rdf" level="info"/>
<root level="warn">
<appender-ref ref="STDOUT"/>
+ <appender-ref ref="logstash"/>
</root>
</configuration>
diff --git a/war/pom.xml b/war/pom.xml
index a7a7b64..638a048 100644
--- a/war/pom.xml
+++ b/war/pom.xml
@@ -40,6 +40,11 @@
<scope>runtime</scope>
</dependency>
<dependency>
+ <groupId>net.logstash.logback</groupId>
+ <artifactId>logstash-logback-encoder</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
<groupId>org.jhades</groupId>
<artifactId>jhades</artifactId>
<scope>runtime</scope>
diff --git a/war/src/main/resources/logback.xml
b/war/src/main/resources/logback.xml
index e13e8a9..2ceb3fb 100644
--- a/war/src/main/resources/logback.xml
+++ b/war/src/main/resources/logback.xml
@@ -7,9 +7,17 @@
</encoder>
</appender>
+ <appender name="logstash"
class="net.logstash.logback.appender.LogstashTcpSocketAppender">
+
<destination>${logstash.host:-localhost}:${logstash.port:-11514}</destination>
+ <encoder class="net.logstash.logback.encoder.LogstashEncoder">
+ <customFields>{"type":"wdqs-blazegraph"}</customFields>
+ </encoder>
+ </appender>
+
<logger name="org.wikidata.query.rdf" level="info"/>
<root level="warn">
<appender-ref ref="STDOUT"/>
+ <appender-ref ref="logstash"/>
</root>
</configuration>
--
To view, visit https://gerrit.wikimedia.org/r/371950
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie9075dfbc34055748976648fa3d06a91fd2d9563
Gerrit-PatchSet: 1
Gerrit-Project: wikidata/query/rdf
Gerrit-Branch: master
Gerrit-Owner: Gehel <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
