jenkins-bot has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/368777 )
Change subject: add user-agent and IP address to logs
......................................................................
add user-agent and IP address to logs
A servlet Filter adds user-agent and IP address to the logging MDC
(Mapped Diagnostic Contexts). The logback configuration is updated to expose
those new fields on each log message.
Change-Id: I7aaa519b7e1137fd572b8a0a5a164896c9be819c
---
M blazegraph/pom.xml
R
blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/filters/ClientIPFilter.java
A
blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/filters/LoggingFilter.java
R
blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/filters/ClientIPFilterTest.java
A
blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/filters/LoggingFilterTest.java
M war/src/main/resources/logback.xml
M war/src/main/webapp/WEB-INF/web.xml
7 files changed, 165 insertions(+), 10 deletions(-)
Approvals:
Smalyshev: Looks good to me, approved
jenkins-bot: Verified
diff --git a/blazegraph/pom.xml b/blazegraph/pom.xml
index 5318026..6cddedd 100644
--- a/blazegraph/pom.xml
+++ b/blazegraph/pom.xml
@@ -116,6 +116,12 @@
<scope>provided</scope>
</dependency>
<dependency>
+ <!-- testing the LoggingFilter requires a "real" logging framework -->
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
<scope>test</scope>
@@ -133,11 +139,6 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-simple</artifactId>
<scope>test</scope>
</dependency>
<dependency>
diff --git
a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/ClientIPFilter.java
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/filters/ClientIPFilter.java
similarity index 97%
rename from
blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/ClientIPFilter.java
rename to
blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/filters/ClientIPFilter.java
index ce2e67e..9d3211c 100644
---
a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/ClientIPFilter.java
+++
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/filters/ClientIPFilter.java
@@ -1,4 +1,4 @@
-package org.wikidata.query.rdf.blazegraph.throttling;
+package org.wikidata.query.rdf.blazegraph.filters;
import javax.servlet.Filter;
import javax.servlet.FilterChain;
diff --git
a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/filters/LoggingFilter.java
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/filters/LoggingFilter.java
new file mode 100644
index 0000000..9e2af29
--- /dev/null
+++
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/filters/LoggingFilter.java
@@ -0,0 +1,63 @@
+package org.wikidata.query.rdf.blazegraph.filters;
+
+import org.slf4j.MDC;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import java.io.Closeable;
+import java.io.IOException;
+
+/**
+ * Add user-agent and IP address to the logging MDC.
+ *
+ * The Mapped Diagnostic Context (MDC) is a way to expose context to the
+ * logging framework, which can then be printed with each log message.
+ *
+ * See <a href="https://logback.qos.ch/manual/mdc.html";>the logback
+ * documentation</a> for details on MDC.
+ */
+public class LoggingFilter implements Filter {
+
+ /** {@inheritDoc} */
+ @Override
+ public void init(FilterConfig filterConfig) throws ServletException {
+ // Do nothing
+ }
+
+ /**
+ * Add user-agent and IP address to the logging MDC.
+ *
+ * @param request
+ * @param response
+ * @param chain
+ * @throws IOException
+ * @throws ServletException
+ */
+ @Override
+ public void doFilter(ServletRequest request, ServletResponse response,
FilterChain chain) throws IOException, ServletException {
+ String remoteAddr = request.getRemoteAddr();
+ String userAgent = null;
+ if (request instanceof HttpServletRequest) {
+ HttpServletRequest httpRequest = (HttpServletRequest) request;
+ userAgent = httpRequest.getHeader("User-Agent");
+ }
+
+ try (
+ Closeable mdcIPAddress = MDC.putCloseable("IPAddress",
remoteAddr);
+ Closeable mdcUserAgent = MDC.putCloseable("UserAgent",
userAgent)
+ ) {
+ chain.doFilter(request, response);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void destroy() {
+ // do nothing
+ }
+}
diff --git
a/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/throttling/ClientIPFilterTest.java
b/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/filters/ClientIPFilterTest.java
similarity index 97%
rename from
blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/throttling/ClientIPFilterTest.java
rename to
blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/filters/ClientIPFilterTest.java
index 7001ac8..56c837c 100644
---
a/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/throttling/ClientIPFilterTest.java
+++
b/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/filters/ClientIPFilterTest.java
@@ -1,4 +1,4 @@
-package org.wikidata.query.rdf.blazegraph.throttling;
+package org.wikidata.query.rdf.blazegraph.filters;
import org.junit.Test;
import org.junit.runner.RunWith;
diff --git
a/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/filters/LoggingFilterTest.java
b/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/filters/LoggingFilterTest.java
new file mode 100644
index 0000000..89e7103
--- /dev/null
+++
b/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/filters/LoggingFilterTest.java
@@ -0,0 +1,84 @@
+package org.wikidata.query.rdf.blazegraph.filters;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+import org.slf4j.MDC;
+import org.springframework.mock.web.MockHttpServletRequest;
+
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import java.io.IOException;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.Assert.assertNull;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+
+@RunWith(MockitoJUnitRunner.class)
+public class LoggingFilterTest {
+
+ private LoggingFilter filter = new LoggingFilter();
+ @Mock private ServletResponse response;
+ @Mock private FilterChain chain;
+
+ @Test
+ public void ipAndUserAgentAreAddedToMDC() throws IOException,
ServletException {
+ MockHttpServletRequest request = new MockHttpServletRequest();
+ request.setRemoteAddr("1.2.3.4");
+ request.addHeader("User-Agent", "UA1");
+
+ doAnswer(invocation -> {
+ assertThat(MDC.get("IPAddress"), equalTo("1.2.3.4"));
+ assertThat(MDC.get("UserAgent"), equalTo("UA1"));
+ return null;
+ }).when(chain).doFilter(request, response);
+
+
+ filter.doFilter(request, response, chain);
+ }
+
+ @Test
+ public void nullIpAndUserAgentDontCauseIssues() throws IOException,
ServletException {
+ MockHttpServletRequest request = new MockHttpServletRequest();
+ request.setRemoteAddr(null);
+ // dont set User-Agent header
+
+ doAnswer(invocation -> {
+ assertNull(MDC.get("IPAddress"));
+ assertNull(MDC.get("UserAgent"));
+ return null;
+ }).when(chain).doFilter(request, response);
+
+
+ filter.doFilter(request, response, chain);
+ }
+
+ @Test
+ public void ipAndUserAgentAreCleanedUp() throws IOException,
ServletException {
+ MockHttpServletRequest request = new MockHttpServletRequest();
+ request.setRemoteAddr("1.2.3.4");
+ request.addHeader("User-Agent", "UA1");
+
+ filter.doFilter(request, response, chain);
+
+ assertNull(MDC.get("IPAddress"));
+ assertNull(MDC.get("UserAgent"));
+ }
+
+ @Test
+ public void nonHttpRequestsDontCauseIssues() throws IOException,
ServletException {
+ ServletRequest request = mock(ServletRequest.class);
+ doAnswer(invocation -> {
+ assertNull(MDC.get("UserAgent"));
+ return null;
+ }).when(chain).doFilter(request, response);
+
+ filter.doFilter(request, response, chain);
+ }
+
+}
diff --git a/war/src/main/resources/logback.xml
b/war/src/main/resources/logback.xml
index 17370c8..e13e8a9 100644
--- a/war/src/main/resources/logback.xml
+++ b/war/src/main/resources/logback.xml
@@ -2,13 +2,12 @@
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
- <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} -
%msg%n</pattern>
+ <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36}
IP:%X{IPAddress} UA:%X{UserAgent} - %msg%n</pattern>
<outputPatternAsHeader>true</outputPatternAsHeader>
</encoder>
</appender>
<logger name="org.wikidata.query.rdf" level="info"/>
- <logger name="org.wikidata.query.rdf.tool.wikibase" level="debug"/>
<root level="warn">
<appender-ref ref="STDOUT"/>
diff --git a/war/src/main/webapp/WEB-INF/web.xml
b/war/src/main/webapp/WEB-INF/web.xml
index b6f4484..d6a33e0 100644
--- a/war/src/main/webapp/WEB-INF/web.xml
+++ b/war/src/main/webapp/WEB-INF/web.xml
@@ -86,13 +86,21 @@
</listener>
<filter>
<filter-name>real-ip-filter</filter-name>
-
<filter-class>org.wikidata.query.rdf.blazegraph.throttling.ClientIPFilter</filter-class>
+
<filter-class>org.wikidata.query.rdf.blazegraph.filters.ClientIPFilter</filter-class>
</filter>
<filter-mapping>
<filter-name>real-ip-filter</filter-name>
<url-pattern>/*</url-pattern>
</filter-mapping>
<filter>
+ <filter-name>logging-filter</filter-name>
+
<filter-class>org.wikidata.query.rdf.blazegraph.filters.LoggingFilter</filter-class>
+ </filter>
+ <filter-mapping>
+ <filter-name>logging-filter</filter-name>
+ <url-pattern>/*</url-pattern>
+ </filter-mapping>
+ <filter>
<filter-name>throttling-filter</filter-name>
<filter-class>org.wikidata.query.rdf.blazegraph.throttling.ThrottlingFilter</filter-class>
</filter>
--
To view, visit https://gerrit.wikimedia.org/r/368777
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I7aaa519b7e1137fd572b8a0a5a164896c9be819c
Gerrit-PatchSet: 5
Gerrit-Project: wikidata/query/rdf
Gerrit-Branch: master
Gerrit-Owner: Gehel <[email protected]>
Gerrit-Reviewer: Faidon Liambotis <[email protected]>
Gerrit-Reviewer: Gehel <[email protected]>
Gerrit-Reviewer: Lucas Werkmeister (WMDE) <[email protected]>
Gerrit-Reviewer: Smalyshev <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
