This is an automated email from the ASF dual-hosted git repository.
atul pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new bb771b30521 Support flexible rotation periods for file-based request
logging (#17976)
bb771b30521 is described below
commit bb771b3052167ea4ceb49573ab8a5d24fd521ee9
Author: Atul Mohan <[email protected]>
AuthorDate: Tue May 6 19:33:22 2025 -0700
Support flexible rotation periods for file-based request logging (#17976)
* Support flexible rotation periods for file-based request logging
* Fix docs
* Fix docs
* Fix docs
* Address comments
* Remove unused variable
---
docs/configuration/index.md | 7 +--
.../apache/druid/server/log/FileRequestLogger.java | 36 +++++++++-----
.../server/log/FileRequestLoggerProvider.java | 7 ++-
.../druid/server/log/FileRequestLoggerTest.java | 55 ++++++++++++++++++++--
4 files changed, 86 insertions(+), 19 deletions(-)
diff --git a/docs/configuration/index.md b/docs/configuration/index.md
index c1f78012347..c7ddc73efa0 100644
--- a/docs/configuration/index.md
+++ b/docs/configuration/index.md
@@ -275,9 +275,10 @@ The `file` request logger stores daily request logs on
disk.
|Property|Description|Default|
|--------|-----------|-------|
-|`druid.request.logging.dir`|Historical, Realtime, and Broker services
maintain request logs of all of the requests they get (interaction is via POST,
so normal request logs don’t generally capture information about the actual
query), this specifies the directory to store the request logs in|none|
-|`druid.request.logging.filePattern`|[Joda datetime
format](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html)
for each file|"yyyy-MM-dd'.log'"|
-| `druid.request.logging.durationToRetain`| Period to retain the request logs
on disk. The period should be at least longer than `P1D`.| none|
+|`druid.request.logging.dir`| Historical, Realtime, and Broker services
maintain request logs of all of the requests they get (interaction is via POST,
so normal request logs don’t generally capture information about the actual
query), this specifies the directory to store the request logs in. | none|
+|`druid.request.logging.filePattern`| [Joda datetime
format](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html)
for each file.| "yyyy-MM-dd'.log'"|
+|`druid.request.logging.durationToRetain`| Period to retain the request logs
on disk. The period should be at least as long as roll period.| none|
+|`druid.request.logging.rollPeriod`| Defines the log rotation period for
request logs. The period should be at least `PT1H`. For periods smaller than 1
day, it is recommended to use `"yyyy-MM-dd-HH'.log'"` as the file pattern.| P1D|
The format of request logs is TSV, one line per requests, with five fields:
timestamp, remote\_addr, native\_query, query\_context, sql\_query.
diff --git
a/server/src/main/java/org/apache/druid/server/log/FileRequestLogger.java
b/server/src/main/java/org/apache/druid/server/log/FileRequestLogger.java
index a4fe842f3cb..b1bf22061ba 100644
--- a/server/src/main/java/org/apache/druid/server/log/FileRequestLogger.java
+++ b/server/src/main/java/org/apache/druid/server/log/FileRequestLogger.java
@@ -57,9 +57,10 @@ public class FileRequestLogger implements RequestLogger
private final File baseDir;
private final DateTimeFormatter filePattern;
private final Duration durationToRetain;
+ private final Duration rollPeriod;
private final Object lock = new Object();
- private DateTime currentDay;
+ private DateTime currentPeriodStart;
private OutputStreamWriter fileWriter;
public FileRequestLogger(
@@ -67,7 +68,8 @@ public class FileRequestLogger implements RequestLogger
ScheduledExecutorService exec,
File baseDir,
String filePattern,
- Duration durationToRetain
+ Duration durationToRetain,
+ Duration rollPeriod
)
{
this.exec = exec;
@@ -75,9 +77,14 @@ public class FileRequestLogger implements RequestLogger
this.baseDir = baseDir;
this.filePattern = DateTimeFormat.forPattern(filePattern);
this.durationToRetain = durationToRetain;
+ this.rollPeriod = rollPeriod;
Preconditions.checkArgument(
- this.durationToRetain == null ||
this.durationToRetain.compareTo(Duration.standardDays(1)) >= 0,
- "request logs retention period must be atleast P1D"
+ this.rollPeriod == null ||
this.rollPeriod.compareTo(Duration.standardHours(1)) >= 0,
+ "request logs roll period must be atleast PT1H"
+ );
+ Preconditions.checkArgument(
+ this.durationToRetain == null ||
this.durationToRetain.compareTo(this.rollPeriod) >= 0,
+ "request logs retention period must be atleast as long as roll period"
);
}
@@ -89,19 +96,24 @@ public class FileRequestLogger implements RequestLogger
FileUtils.mkdirp(baseDir);
MutableDateTime mutableDateTime =
DateTimes.nowUtc().toMutableDateTime(ISOChronology.getInstanceUTC());
- mutableDateTime.setMillisOfDay(0);
+ mutableDateTime.setMinuteOfHour(0);
+ mutableDateTime.setSecondOfMinute(0);
+ mutableDateTime.setMillisOfSecond(0);
+ if (rollPeriod.compareTo(Duration.standardDays(1)) >= 0) {
+ mutableDateTime.setHourOfDay(0);
+ }
synchronized (lock) {
- currentDay =
mutableDateTime.toDateTime(ISOChronology.getInstanceUTC());
+ currentPeriodStart =
mutableDateTime.toDateTime(ISOChronology.getInstanceUTC());
fileWriter = getFileWriter();
}
- long nextDay = currentDay.plusDays(1).getMillis();
- Duration initialDelay = new Duration(nextDay -
System.currentTimeMillis());
+ long nextPeriodMillis = currentPeriodStart.plus(rollPeriod).getMillis();
+ Duration initialDelay = new Duration(nextPeriodMillis -
System.currentTimeMillis());
ScheduledExecutors.scheduleWithFixedDelay(
exec,
initialDelay,
- Duration.standardDays(1),
+ rollPeriod,
new Callable<>()
{
@Override
@@ -109,11 +121,11 @@ public class FileRequestLogger implements RequestLogger
{
try {
synchronized (lock) {
- currentDay = currentDay.plusDays(1);
+ currentPeriodStart = currentPeriodStart.plus(rollPeriod);
CloseableUtils.closeAndSuppressExceptions(
fileWriter,
- e -> log.warn("Could not close log file for %s. Creating
new log file anyway.", currentDay)
+ e -> log.warn("Could not close log file for %s. Creating
new log file anyway.", currentPeriodStart)
);
fileWriter = getFileWriter();
@@ -173,7 +185,7 @@ public class FileRequestLogger implements RequestLogger
private OutputStreamWriter getFileWriter() throws FileNotFoundException
{
return new OutputStreamWriter(
- new FileOutputStream(new File(baseDir, filePattern.print(currentDay)),
true),
+ new FileOutputStream(new File(baseDir,
filePattern.print(currentPeriodStart)), true),
StandardCharsets.UTF_8
);
}
diff --git
a/server/src/main/java/org/apache/druid/server/log/FileRequestLoggerProvider.java
b/server/src/main/java/org/apache/druid/server/log/FileRequestLoggerProvider.java
index 9765929fd3b..cb5f8bb1c26 100644
---
a/server/src/main/java/org/apache/druid/server/log/FileRequestLoggerProvider.java
+++
b/server/src/main/java/org/apache/druid/server/log/FileRequestLoggerProvider.java
@@ -27,6 +27,7 @@ import org.apache.druid.guice.annotations.Json;
import org.apache.druid.java.util.common.concurrent.ScheduledExecutorFactory;
import org.apache.druid.java.util.common.logger.Logger;
import org.joda.time.Duration;
+import org.joda.time.Period;
import javax.validation.constraints.NotNull;
import java.io.File;
@@ -58,6 +59,9 @@ public class FileRequestLoggerProvider implements
RequestLoggerProvider
@JsonProperty
private Duration durationToRetain;
+ @JsonProperty
+ private Duration rollPeriod = new Period("P1D").toStandardDuration();
+
@Override
public RequestLogger get()
{
@@ -66,7 +70,8 @@ public class FileRequestLoggerProvider implements
RequestLoggerProvider
factory.create(1, "RequestLogger-%s"),
dir,
filePattern,
- durationToRetain
+ durationToRetain,
+ rollPeriod
);
log.debug(new Exception("Stack trace"), "Creating %s at", logger);
return logger;
diff --git
a/server/src/test/java/org/apache/druid/server/log/FileRequestLoggerTest.java
b/server/src/test/java/org/apache/druid/server/log/FileRequestLoggerTest.java
index abbb7b95600..0ceb539adab 100644
---
a/server/src/test/java/org/apache/druid/server/log/FileRequestLoggerTest.java
+++
b/server/src/test/java/org/apache/druid/server/log/FileRequestLoggerTest.java
@@ -64,7 +64,8 @@ public class FileRequestLoggerTest
scheduler,
logDir,
"yyyy-MM-dd'.log'",
- null
+ null,
+ Duration.standardDays(1)
);
fileRequestLogger.start();
@@ -101,6 +102,7 @@ public class FileRequestLoggerTest
scheduler,
logDir,
"yyyy-MM-dd'.log'",
+ Duration.standardDays(1),
Duration.standardDays(1)
);
fileRequestLogger.start();
@@ -119,7 +121,7 @@ public class FileRequestLoggerTest
public void testLogRemoveWithInvalidDuration() throws Exception
{
expectedException.expect(IllegalArgumentException.class);
- expectedException.expectMessage("request logs retention period must be
atleast P1D");
+ expectedException.expectMessage("request logs retention period must be
atleast as long as roll period");
ObjectMapper objectMapper = new ObjectMapper();
File logDir = temporaryFolder.newFolder();
FileRequestLogger fileRequestLogger = new FileRequestLogger(
@@ -127,7 +129,54 @@ public class FileRequestLoggerTest
scheduler,
logDir,
"yyyy-MM-dd'.log'",
- Duration.standardHours(12)
+ Duration.standardMinutes(30),
+ Duration.standardDays(1)
);
}
+
+ @Test
+ public void testRollPeriodForLog() throws Exception
+ {
+ ObjectMapper objectMapper = new ObjectMapper();
+ DateTime dateTime = DateTimes.nowUtc();
+ File logDir = temporaryFolder.newFolder();
+ String sqlQueryLogString = dateTime + "\t" + HOST + "\t" + "sql";
+
+ FileRequestLogger hourlyLogger = new FileRequestLogger(
+ objectMapper,
+ scheduler,
+ logDir,
+ "yyyy-MM-dd-HH'.log'",
+ null,
+ Duration.standardHours(1)
+ );
+ FileRequestLogger dailyLoggerWithHourPattern = new FileRequestLogger(
+ objectMapper,
+ scheduler,
+ logDir,
+ "yyyy-MM-dd-HH'.log'",
+ null,
+ Duration.standardHours(26)
+ );
+ hourlyLogger.start();
+ dailyLoggerWithHourPattern.start();
+
+ RequestLogLine sqlRequestLogLine =
EasyMock.createMock(RequestLogLine.class);
+
EasyMock.expect(sqlRequestLogLine.getSqlQueryLine(EasyMock.anyObject())).andReturn(sqlQueryLogString).anyTimes();
+ EasyMock.replay(sqlRequestLogLine);
+
+ hourlyLogger.logSqlQuery(sqlRequestLogLine);
+ dailyLoggerWithHourPattern.logSqlQuery(sqlRequestLogLine);
+
+ File hourlyLogFile = new File(logDir,
dateTime.toString("yyyy-MM-dd-HH'.log'"));
+ // The hour is zeroed out since the roll period is more than 1 day
+ File dailyLogFile = new File(logDir,
dateTime.toString("yyyy-MM-dd-00'.log'"));
+ String hourlyLogString =
CharStreams.toString(Files.newBufferedReader(hourlyLogFile.toPath(),
StandardCharsets.UTF_8));
+ String dailyLogString =
CharStreams.toString(Files.newBufferedReader(dailyLogFile.toPath(),
StandardCharsets.UTF_8));
+ Assert.assertTrue(hourlyLogString.contains(sqlQueryLogString + "\n"));
+ Assert.assertTrue(dailyLogString.contains(sqlQueryLogString + "\n"));
+
+ hourlyLogger.stop();
+ dailyLoggerWithHourPattern.stop();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
