This is an automated email from the ASF dual-hosted git repository.
atul pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 93a9a4b1c5 Add retention for file request logs (#12559)
93a9a4b1c5 is described below
commit 93a9a4b1c5342d18abf269202d1c777852b6532a
Author: Atul Mohan <[email protected]>
AuthorDate: Wed Jul 27 08:17:02 2022 -0700
Add retention for file request logs (#12559)
* Add retention for file request logs
* Spelling
---
docs/configuration/index.md | 1 +
.../apache/druid/server/log/FileRequestLogger.java | 55 +++++++++++++++++-
.../server/log/FileRequestLoggerProvider.java | 7 ++-
.../druid/server/log/FileRequestLoggerTest.java | 67 +++++++++++++++++++++-
4 files changed, 123 insertions(+), 7 deletions(-)
diff --git a/docs/configuration/index.md b/docs/configuration/index.md
index 32cedc5e7f..50ff571cf0 100644
--- a/docs/configuration/index.md
+++ b/docs/configuration/index.md
@@ -267,6 +267,7 @@ The `file` request logger stores daily request logs on disk.
|--------|-----------|-------|
|`druid.request.logging.dir`|Historical, Realtime and Broker processes
maintain request logs of all of the requests they get (interaction is via POST,
so normal request logs don’t generally capture information about the actual
query), this specifies the directory to store the request logs in|none|
|`druid.request.logging.filePattern`|[Joda datetime
format](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html)
for each file|"yyyy-MM-dd'.log'"|
+| `druid.request.logging.durationToRetain`| Period to retain the request logs
on disk. The period should be at least longer than `P1D`.| none
The format of request logs is TSV, one line per requests, with five fields:
timestamp, remote\_addr, native\_query, query\_context, sql\_query.
diff --git
a/server/src/main/java/org/apache/druid/server/log/FileRequestLogger.java
b/server/src/main/java/org/apache/druid/server/log/FileRequestLogger.java
index 9ca4516e94..35a10d9a8f 100644
--- a/server/src/main/java/org/apache/druid/server/log/FileRequestLogger.java
+++ b/server/src/main/java/org/apache/druid/server/log/FileRequestLogger.java
@@ -20,6 +20,7 @@
package org.apache.druid.server.log;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Preconditions;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.FileUtils;
import org.apache.druid.java.util.common.concurrent.ScheduledExecutors;
@@ -45,7 +46,7 @@ import java.util.concurrent.Callable;
import java.util.concurrent.ScheduledExecutorService;
/**
- *
+ * Request logger implementation that logs query requests and writes them to a
file.
*/
public class FileRequestLogger implements RequestLogger
{
@@ -55,18 +56,29 @@ public class FileRequestLogger implements RequestLogger
private final ScheduledExecutorService exec;
private final File baseDir;
private final DateTimeFormatter filePattern;
-
+ private final Duration durationToRetain;
private final Object lock = new Object();
private DateTime currentDay;
private OutputStreamWriter fileWriter;
- public FileRequestLogger(ObjectMapper objectMapper, ScheduledExecutorService
exec, File baseDir, String filePattern)
+ public FileRequestLogger(
+ ObjectMapper objectMapper,
+ ScheduledExecutorService exec,
+ File baseDir,
+ String filePattern,
+ Duration durationToRetain
+ )
{
this.exec = exec;
this.objectMapper = objectMapper;
this.baseDir = baseDir;
this.filePattern = DateTimeFormat.forPattern(filePattern);
+ this.durationToRetain = durationToRetain;
+ Preconditions.checkArgument(
+ this.durationToRetain == null ||
this.durationToRetain.compareTo(Duration.standardDays(1)) >= 0,
+ "request logs retention period must be atleast P1D"
+ );
}
@LifecycleStart
@@ -115,12 +127,49 @@ public class FileRequestLogger implements RequestLogger
}
}
);
+ if (durationToRetain != null) {
+ ScheduledExecutors.scheduleWithFixedDelay(
+ exec,
+ new Duration(0),
+ Duration.standardDays(1),
+ new Runnable()
+ {
+ @Override
+ public void run()
+ {
+ try {
+ long timestamp = System.currentTimeMillis() -
durationToRetain.getMillis();
+ removeFilesOlderThan(timestamp);
+ }
+ catch (Exception ex) {
+ log.error(ex, "Failed to clean-up the request logs");
+ }
+ }
+ }
+ );
+ }
}
catch (IOException e) {
throw new RuntimeException(e);
}
}
+ private void removeFilesOlderThan(long timestamp) throws IOException
+ {
+ File[] files = baseDir.listFiles(f -> f.lastModified() < timestamp);
+ if (files != null) {
+ for (File file : files) {
+ log.info("Deleting request log [%s].", file.getAbsolutePath());
+ org.apache.commons.io.FileUtils.forceDelete(file);
+ if (Thread.currentThread().isInterrupted()) {
+ throw new IOException(
+ new InterruptedException("Thread interrupted. Couldn't delete
all tasklogs.")
+ );
+ }
+ }
+ }
+ }
+
private OutputStreamWriter getFileWriter() throws FileNotFoundException
{
return new OutputStreamWriter(
diff --git
a/server/src/main/java/org/apache/druid/server/log/FileRequestLoggerProvider.java
b/server/src/main/java/org/apache/druid/server/log/FileRequestLoggerProvider.java
index 5c1de5f64a..9765929fd3 100644
---
a/server/src/main/java/org/apache/druid/server/log/FileRequestLoggerProvider.java
+++
b/server/src/main/java/org/apache/druid/server/log/FileRequestLoggerProvider.java
@@ -26,6 +26,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.druid.guice.annotations.Json;
import org.apache.druid.java.util.common.concurrent.ScheduledExecutorFactory;
import org.apache.druid.java.util.common.logger.Logger;
+import org.joda.time.Duration;
import javax.validation.constraints.NotNull;
import java.io.File;
@@ -54,6 +55,9 @@ public class FileRequestLoggerProvider implements
RequestLoggerProvider
@Json
private ObjectMapper jsonMapper = null;
+ @JsonProperty
+ private Duration durationToRetain;
+
@Override
public RequestLogger get()
{
@@ -61,7 +65,8 @@ public class FileRequestLoggerProvider implements
RequestLoggerProvider
jsonMapper,
factory.create(1, "RequestLogger-%s"),
dir,
- filePattern
+ filePattern,
+ durationToRetain
);
log.debug(new Exception("Stack trace"), "Creating %s at", logger);
return logger;
diff --git
a/server/src/test/java/org/apache/druid/server/log/FileRequestLoggerTest.java
b/server/src/test/java/org/apache/druid/server/log/FileRequestLoggerTest.java
index 726b5f8e79..d0668d7df0 100644
---
a/server/src/test/java/org/apache/druid/server/log/FileRequestLoggerTest.java
+++
b/server/src/test/java/org/apache/druid/server/log/FileRequestLoggerTest.java
@@ -25,14 +25,17 @@ import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.server.RequestLogLine;
import org.easymock.EasyMock;
import org.joda.time.DateTime;
+import org.joda.time.Duration;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder;
import java.io.File;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
+import java.util.Date;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@@ -44,7 +47,11 @@ public class FileRequestLoggerTest
@Rule
public TemporaryFolder temporaryFolder = new TemporaryFolder();
- @Test public void testLog() throws Exception
+ @Rule
+ public ExpectedException expectedException = ExpectedException.none();
+
+ @Test
+ public void testLog() throws Exception
{
ObjectMapper objectMapper = new ObjectMapper();
DateTime dateTime = DateTimes.nowUtc();
@@ -52,11 +59,19 @@ public class FileRequestLoggerTest
String nativeQueryLogString = dateTime + "\t" + HOST + "\t" + "native";
String sqlQueryLogString = dateTime + "\t" + HOST + "\t" + "sql";
- FileRequestLogger fileRequestLogger = new FileRequestLogger(objectMapper,
scheduler, logDir, "yyyy-MM-dd'.log'");
+ FileRequestLogger fileRequestLogger = new FileRequestLogger(
+ objectMapper,
+ scheduler,
+ logDir,
+ "yyyy-MM-dd'.log'",
+ null
+ );
fileRequestLogger.start();
RequestLogLine nativeRequestLogLine =
EasyMock.createMock(RequestLogLine.class);
-
EasyMock.expect(nativeRequestLogLine.getNativeQueryLine(EasyMock.anyObject())).andReturn(nativeQueryLogString).anyTimes();
+
EasyMock.expect(nativeRequestLogLine.getNativeQueryLine(EasyMock.anyObject()))
+ .andReturn(nativeQueryLogString)
+ .anyTimes();
RequestLogLine sqlRequestLogLine =
EasyMock.createMock(RequestLogLine.class);
EasyMock.expect(sqlRequestLogLine.getSqlQueryLine(EasyMock.anyObject())).andReturn(sqlQueryLogString).anyTimes();
EasyMock.replay(nativeRequestLogLine, sqlRequestLogLine);
@@ -69,4 +84,50 @@ public class FileRequestLoggerTest
Assert.assertTrue(logString.contains(nativeQueryLogString + "\n" +
sqlQueryLogString + "\n"));
fileRequestLogger.stop();
}
+
+ @Test
+ public void testLogRemove() throws Exception
+ {
+ ObjectMapper objectMapper = new ObjectMapper();
+ File logDir = temporaryFolder.newFolder();
+ DateTime dateTime = DateTimes.nowUtc();
+ String logString = dateTime + "\t" + HOST + "\t" + "logString";
+
+ File oldLogFile = new File(logDir, "2000-01-01.log");
+ com.google.common.io.Files.write("testOldLogContent", oldLogFile,
StandardCharsets.UTF_8);
+ oldLogFile.setLastModified(new Date(0).getTime());
+ FileRequestLogger fileRequestLogger = new FileRequestLogger(
+ objectMapper,
+ scheduler,
+ logDir,
+ "yyyy-MM-dd'.log'",
+ Duration.standardDays(1)
+ );
+ fileRequestLogger.start();
+ RequestLogLine nativeRequestLogLine =
EasyMock.createMock(RequestLogLine.class);
+
EasyMock.expect(nativeRequestLogLine.getNativeQueryLine(EasyMock.anyObject())).andReturn(logString).anyTimes();
+ EasyMock.replay(nativeRequestLogLine);
+ fileRequestLogger.logNativeQuery(nativeRequestLogLine);
+ File logFile = new File(logDir, dateTime.toString("yyyy-MM-dd'.log'"));
+ Thread.sleep(100);
+ Assert.assertFalse(oldLogFile.exists());
+ Assert.assertTrue(logFile.exists());
+ fileRequestLogger.stop();
+ }
+
+ @Test
+ public void testLogRemoveWithInvalidDuration() throws Exception
+ {
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("request logs retention period must be
atleast P1D");
+ ObjectMapper objectMapper = new ObjectMapper();
+ File logDir = temporaryFolder.newFolder();
+ FileRequestLogger fileRequestLogger = new FileRequestLogger(
+ objectMapper,
+ scheduler,
+ logDir,
+ "yyyy-MM-dd'.log'",
+ Duration.standardHours(12)
+ );
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
