This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new e252abe Enable toggling request logging on/off for different query
types (#7562)
e252abe is described below
commit e252abedc5d7bd20c3c1dad7daf4cbdeb99ec761
Author: Lucas Capistrant <[email protected]>
AuthorDate: Tue Aug 6 07:47:30 2019 -0500
Enable toggling request logging on/off for different query types (#7562)
* Enable ability to toggle SegmentMetadata request logging on/off
* Move SegmentMetadata query log filter to FilteredRequestLogger
* Update documentation to reflect the segment metadata flag moving to the
filtered request logger
* Modify patch to allow blacklist of query types to not log to request
logger
* Address styling and naming requests following latest code review
* Fix indentation on multiple locations per Druid style rules
---
docs/content/configuration/index.md | 1 +
.../server/log/FileRequestLoggerProvider.java | 7 ++-
.../server/log/FilteredRequestLoggerProvider.java | 30 ++++++++--
.../server/log/FilteredRequestLoggerTest.java | 65 +++++++++++++++++++++-
4 files changed, 96 insertions(+), 7 deletions(-)
diff --git a/docs/content/configuration/index.md
b/docs/content/configuration/index.md
index 0e9e663..fcccac7 100644
--- a/docs/content/configuration/index.md
+++ b/docs/content/configuration/index.md
@@ -378,6 +378,7 @@ For native query, only request logs where query/time is
above the threshold are
|--------|-----------|-------|
|`druid.request.logging.queryTimeThresholdMs`|Threshold value for query/time
in milliseconds.|0 i.e no filtering|
|`druid.request.logging.sqlQueryTimeThresholdMs`|Threshold value for
sqlQuery/time in milliseconds.|0 i.e no filtering|
+|`druid.request.logging.mutedQueryTypes` | Query requests of these types are
not logged. Query types are defined as string objects corresponding to the
"queryType" value for the specified query in the Druid's [native JSON query
API](http://druid.apache.org/docs/latest/querying/querying.html). Misspelled
query types will be ignored. Example to ignore scan and timeBoundary queries:
["scan", "timeBoundary"]| []|
|`druid.request.logging.delegate.type`|Type of delegate request logger to log
requests.|none|
#### Composite Request Logging
diff --git
a/server/src/main/java/org/apache/druid/server/log/FileRequestLoggerProvider.java
b/server/src/main/java/org/apache/druid/server/log/FileRequestLoggerProvider.java
index be88caf..0d993d6 100644
---
a/server/src/main/java/org/apache/druid/server/log/FileRequestLoggerProvider.java
+++
b/server/src/main/java/org/apache/druid/server/log/FileRequestLoggerProvider.java
@@ -58,7 +58,12 @@ public class FileRequestLoggerProvider implements
RequestLoggerProvider
@Override
public RequestLogger get()
{
- FileRequestLogger logger = new FileRequestLogger(jsonMapper,
factory.create(1, "RequestLogger-%s"), dir, filePattern);
+ FileRequestLogger logger = new FileRequestLogger(
+ jsonMapper,
+ factory.create(1, "RequestLogger-%s"),
+ dir,
+ filePattern
+ );
log.debug(new Exception("Stack trace"), "Creating %s at", logger);
return logger;
}
diff --git
a/server/src/main/java/org/apache/druid/server/log/FilteredRequestLoggerProvider.java
b/server/src/main/java/org/apache/druid/server/log/FilteredRequestLoggerProvider.java
index 3bee5f4..beee88e 100644
---
a/server/src/main/java/org/apache/druid/server/log/FilteredRequestLoggerProvider.java
+++
b/server/src/main/java/org/apache/druid/server/log/FilteredRequestLoggerProvider.java
@@ -21,13 +21,16 @@ package org.apache.druid.server.log;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.collect.ImmutableList;
import org.apache.druid.java.util.common.lifecycle.LifecycleStart;
import org.apache.druid.java.util.common.lifecycle.LifecycleStop;
import org.apache.druid.java.util.common.logger.Logger;
+import org.apache.druid.query.Query;
import org.apache.druid.server.RequestLogLine;
import javax.validation.constraints.NotNull;
import java.io.IOException;
+import java.util.List;
/**
*/
@@ -46,13 +49,17 @@ public class FilteredRequestLoggerProvider implements
RequestLoggerProvider
@JsonProperty
private long sqlQueryTimeThresholdMs = 0;
+ @JsonProperty
+ private List<String> mutedQueryTypes = ImmutableList.of();
+
@Override
public RequestLogger get()
{
FilteredRequestLogger logger = new FilteredRequestLogger(
delegate.get(),
queryTimeThresholdMs,
- sqlQueryTimeThresholdMs
+ sqlQueryTimeThresholdMs,
+ mutedQueryTypes
);
log.debug(new Exception("Stack trace"), "Creating %s at", logger);
return logger;
@@ -63,12 +70,19 @@ public class FilteredRequestLoggerProvider implements
RequestLoggerProvider
private final RequestLogger logger;
private final long queryTimeThresholdMs;
private final long sqlQueryTimeThresholdMs;
-
- public FilteredRequestLogger(RequestLogger logger, long
queryTimeThresholdMs, long sqlQueryTimeThresholdMs)
+ private final List<String> mutedQueryTypes;
+
+ public FilteredRequestLogger(
+ RequestLogger logger,
+ long queryTimeThresholdMs,
+ long sqlQueryTimeThresholdMs,
+ List<String> mutedQueryTypes
+ )
{
this.logger = logger;
this.queryTimeThresholdMs = queryTimeThresholdMs;
this.sqlQueryTimeThresholdMs = sqlQueryTimeThresholdMs;
+ this.mutedQueryTypes = mutedQueryTypes;
}
public long getQueryTimeThresholdMs()
@@ -100,6 +114,10 @@ public class FilteredRequestLoggerProvider implements
RequestLoggerProvider
{
Object queryTime =
requestLogLine.getQueryStats().getStats().get("query/time");
if (queryTime != null && ((Number) queryTime).longValue() >=
queryTimeThresholdMs) {
+ Query query = requestLogLine.getQuery();
+ if (query != null && mutedQueryTypes.contains(query.getType())) {
+ return;
+ }
logger.logNativeQuery(requestLogLine);
}
}
@@ -109,6 +127,10 @@ public class FilteredRequestLoggerProvider implements
RequestLoggerProvider
{
Object sqlQueryTime =
requestLogLine.getQueryStats().getStats().get("sqlQuery/time");
if (sqlQueryTime != null && ((Number) sqlQueryTime).longValue() >=
sqlQueryTimeThresholdMs) {
+ Query query = requestLogLine.getQuery();
+ if (query != null && mutedQueryTypes.contains(query.getType())) {
+ return;
+ }
logger.logSqlQuery(requestLogLine);
}
}
@@ -120,8 +142,8 @@ public class FilteredRequestLoggerProvider implements
RequestLoggerProvider
"logger=" + logger +
", queryTimeThresholdMs=" + queryTimeThresholdMs +
", sqlQueryTimeThresholdMs=" + sqlQueryTimeThresholdMs +
+ ", mutedQueryTypes=" + mutedQueryTypes +
'}';
}
}
-
}
diff --git
a/server/src/test/java/org/apache/druid/server/log/FilteredRequestLoggerTest.java
b/server/src/test/java/org/apache/druid/server/log/FilteredRequestLoggerTest.java
index 13450b1..75f4473 100644
---
a/server/src/test/java/org/apache/druid/server/log/FilteredRequestLoggerTest.java
+++
b/server/src/test/java/org/apache/druid/server/log/FilteredRequestLoggerTest.java
@@ -21,10 +21,14 @@ package org.apache.druid.server.log;
import com.fasterxml.jackson.databind.InjectableValues;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.inject.ProvisionException;
import org.apache.druid.guice.JsonConfigurator;
import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.query.LegacyDataSource;
+import org.apache.druid.query.Query;
+import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery;
import org.apache.druid.server.QueryStats;
import org.apache.druid.server.RequestLogLine;
import org.easymock.EasyMock;
@@ -42,6 +46,16 @@ public class FilteredRequestLoggerTest
@Rule
public final ExpectedException expectedException = ExpectedException.none();
private final DefaultObjectMapper mapper = new DefaultObjectMapper();
+ private final SegmentMetadataQuery testSegmentMetadataQuery = new
SegmentMetadataQuery(
+ new LegacyDataSource("foo"),
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null
+ );
public FilteredRequestLoggerTest()
{
@@ -69,12 +83,14 @@ public class FilteredRequestLoggerTest
FilteredRequestLoggerProvider.FilteredRequestLogger logger = new
FilteredRequestLoggerProvider.FilteredRequestLogger(
delegate,
1000,
- 2000
+ 2000,
+ ImmutableList.of()
);
RequestLogLine nativeRequestLogLine =
EasyMock.createMock(RequestLogLine.class);
EasyMock.expect(nativeRequestLogLine.getQueryStats())
.andReturn(new QueryStats(ImmutableMap.of("query/time", 100)))
.once();
+
RequestLogLine sqlRequestLogLine =
EasyMock.createMock(RequestLogLine.class);
EasyMock.expect(sqlRequestLogLine.getQueryStats())
.andReturn(new QueryStats(ImmutableMap.of("sqlQuery/time", 1000)));
@@ -96,7 +112,8 @@ public class FilteredRequestLoggerTest
FilteredRequestLoggerProvider.FilteredRequestLogger logger = new
FilteredRequestLoggerProvider.FilteredRequestLogger(
delegate,
1000,
- 2000
+ 2000,
+ ImmutableList.of()
);
RequestLogLine nativeRequestLogLine =
EasyMock.createMock(RequestLogLine.class);
@@ -106,6 +123,9 @@ public class FilteredRequestLoggerTest
EasyMock.expect(nativeRequestLogLine.getQueryStats())
.andReturn(new QueryStats(ImmutableMap.of("query/time", 1000)))
.once();
+ EasyMock.expect(nativeRequestLogLine.getQuery())
+ .andReturn(testSegmentMetadataQuery)
+ .times(2);
RequestLogLine sqlRequestLogLine =
EasyMock.createMock(RequestLogLine.class);
EasyMock.expect(sqlRequestLogLine.getQueryStats())
@@ -114,6 +134,9 @@ public class FilteredRequestLoggerTest
EasyMock.expect(sqlRequestLogLine.getQueryStats())
.andReturn(new QueryStats(ImmutableMap.of("sqlQuery/time", 2000)))
.once();
+ EasyMock.expect(sqlRequestLogLine.getQuery())
+ .andReturn(testSegmentMetadataQuery)
+ .times(2);
EasyMock.replay(nativeRequestLogLine, sqlRequestLogLine, delegate);
@@ -126,6 +149,44 @@ public class FilteredRequestLoggerTest
}
@Test
+ public void testNotFilterAboveThresholdSkipSegmentMetadata() throws
IOException
+ {
+ RequestLogger delegate = EasyMock.createStrictMock(RequestLogger.class);
+ delegate.logNativeQuery(EasyMock.anyObject());
+ EasyMock.expectLastCall().andThrow(new IOException());
+ delegate.logSqlQuery(EasyMock.anyObject());
+ EasyMock.expectLastCall().andThrow(new IOException());
+
+ FilteredRequestLoggerProvider.FilteredRequestLogger logger = new
FilteredRequestLoggerProvider.FilteredRequestLogger(
+ delegate,
+ 1000,
+ 2000,
+ ImmutableList.of(Query.SEGMENT_METADATA)
+ );
+
+ RequestLogLine nativeRequestLogLine =
EasyMock.createMock(RequestLogLine.class);
+ EasyMock.expect(nativeRequestLogLine.getQueryStats())
+ .andReturn(new QueryStats(ImmutableMap.of("query/time", 10000)))
+ .once();
+ EasyMock.expect(nativeRequestLogLine.getQuery())
+ .andReturn(testSegmentMetadataQuery)
+ .once();
+
+ RequestLogLine sqlRequestLogLine =
EasyMock.createMock(RequestLogLine.class);
+ EasyMock.expect(sqlRequestLogLine.getQueryStats())
+ .andReturn(new QueryStats(ImmutableMap.of("sqlQuery/time", 10000)))
+ .once();
+ EasyMock.expect(sqlRequestLogLine.getQuery())
+ .andReturn(testSegmentMetadataQuery)
+ .once();
+
+ EasyMock.replay(nativeRequestLogLine, sqlRequestLogLine, delegate);
+
+ logger.logNativeQuery(nativeRequestLogLine);
+ logger.logSqlQuery(sqlRequestLogLine);
+ }
+
+ @Test
public void testConfiguration()
{
final Properties properties = new Properties();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
