This is an automated email from the ASF dual-hosted git repository.
rmani pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new 19ed841 RANGER-3603:HDFS audit files rollover improvement to trigger
rollover in monitoring thread
19ed841 is described below
commit 19ed8415cd2cd26bb07d3aa7d503ae036698c005
Author: Ramesh Mani <[email protected]>
AuthorDate: Fri Feb 25 12:09:12 2022 -0800
RANGER-3603:HDFS audit files rollover improvement to trigger rollover in
monitoring thread
Signed-off-by: Ramesh Mani <[email protected]>
---
.../audit/utils/AbstractRangerAuditWriter.java | 2 +-
.../ranger/audit/utils/RangerJSONAuditWriter.java | 74 +++++++++++++++++++++-
2 files changed, 74 insertions(+), 2 deletions(-)
diff --git
a/agents-audit/src/main/java/org/apache/ranger/audit/utils/AbstractRangerAuditWriter.java
b/agents-audit/src/main/java/org/apache/ranger/audit/utils/AbstractRangerAuditWriter.java
index 2e8e71d..17a7fb9 100644
---
a/agents-audit/src/main/java/org/apache/ranger/audit/utils/AbstractRangerAuditWriter.java
+++
b/agents-audit/src/main/java/org/apache/ranger/audit/utils/AbstractRangerAuditWriter.java
@@ -216,7 +216,7 @@ public abstract class AbstractRangerAuditWriter implements
RangerAuditWriter {
return;
}
- if ( System.currentTimeMillis() > nextRollOverTime.getTime() ) {
+ if ( System.currentTimeMillis() >= nextRollOverTime.getTime() ) {
logger.info("Closing file. Rolling over. name=" + auditProviderName
+ ", fileName=" + currentFileName);
try {
diff --git
a/agents-audit/src/main/java/org/apache/ranger/audit/utils/RangerJSONAuditWriter.java
b/agents-audit/src/main/java/org/apache/ranger/audit/utils/RangerJSONAuditWriter.java
index eb3ea77..755b76d 100644
---
a/agents-audit/src/main/java/org/apache/ranger/audit/utils/RangerJSONAuditWriter.java
+++
b/agents-audit/src/main/java/org/apache/ranger/audit/utils/RangerJSONAuditWriter.java
@@ -26,6 +26,10 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.PrintWriter;
import java.security.PrivilegedExceptionAction;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
import java.util.Collection;
import java.util.Map;
import java.util.Properties;
@@ -36,15 +40,44 @@ import java.util.Properties;
public class RangerJSONAuditWriter extends AbstractRangerAuditWriter {
private static final Logger logger =
LoggerFactory.getLogger(RangerJSONAuditWriter.class);
+ public static final String PROP_HDFS_ROLLOVER_ENABLE_PERIODIC_ROLLOVER
= "file.rollover.enable.periodic.rollover";
+ public static final String PROP_HDFS_ROLLOVER_PERIODIC_ROLLOVER_CHECK_TIME
= "file.rollover.periodic.rollover.check.sec";
protected String JSON_FILE_EXTENSION = ".log";
+ /*
+ * When enableAuditFilePeriodicRollOver is enabled, Audit File in HDFS
would be closed by the defined period in
+ * xasecure.audit.destination.hdfs.file.rollover.sec. By default
xasecure.audit.destination.hdfs.file.rollover.sec = 86400 sec
+ * and file will be closed midnight. Custom rollover time can be set by
defining file.rollover.sec to desire time in seconds.
+ */
+ private boolean enableAuditFilePeriodicRollOver = false;
+
+ /*
+ Time frequency of next occurrence of periodic rollover check. By Default
every 60 seconds the check is done.
+ */
+ private long periodicRollOverCheckTimeinSec;
+
public void init(Properties props, String propPrefix, String
auditProviderName, Map<String,String> auditConfigs) {
if (logger.isDebugEnabled()) {
logger.debug("==> RangerJSONAuditWriter.init()");
}
init();
super.init(props,propPrefix,auditProviderName,auditConfigs);
+
+ // start AuditFilePeriodicRollOverTask if enabled.
+ enableAuditFilePeriodicRollOver = MiscUtil.getBooleanProperty(props,
propPrefix + "." + PROP_HDFS_ROLLOVER_ENABLE_PERIODIC_ROLLOVER, false);
+ if (enableAuditFilePeriodicRollOver) {
+ periodicRollOverCheckTimeinSec = MiscUtil.getLongProperty(props,
propPrefix + "." + PROP_HDFS_ROLLOVER_PERIODIC_ROLLOVER_CHECK_TIME, 60L);
+ try {
+ if (logger.isDebugEnabled()) {
+ logger.debug("rolloverPeriod: " + rolloverPeriod + "
nextRollOverTime: " + nextRollOverTime + " periodicRollOverTimeinSec: " +
periodicRollOverCheckTimeinSec);
+ }
+ startAuditFilePeriodicRollOverTask();
+ } catch (Exception e) {
+ logger.warn("Error enabling audit file perodic rollover..!
Default behavior will be");
+ }
+ }
+
if (logger.isDebugEnabled()) {
logger.debug("<== RangerJSONAuditWriter.init()");
}
@@ -128,7 +161,11 @@ public class RangerJSONAuditWriter extends
AbstractRangerAuditWriter {
}
synchronized public PrintWriter getLogFileStream() throws Exception {
- closeFileIfNeeded();
+ if (!enableAuditFilePeriodicRollOver) {
+ // when periodic rollover is enabled closing of file is done by
the file rollover monitoring task and hence don't need to
+ // close the file inline with audit logging.
+ closeFileIfNeeded();
+ }
// Either there are no open log file or the previous one has been
rolled
// over
PrintWriter logWriter = createWriter();
@@ -171,4 +208,39 @@ public class RangerJSONAuditWriter extends
AbstractRangerAuditWriter {
logger.debug("<== JSONWriter.stop()");
}
}
+
+ private void startAuditFilePeriodicRollOverTask() {
+ ScheduledExecutorService executorService =
Executors.newSingleThreadScheduledExecutor(new
AuditFilePeriodicRollOverTaskThreadFactory());
+
+ if (logger.isDebugEnabled()) {
+
logger.debug("HDFSAuditDestination.startAuditFilePeriodicRollOverTask()
strated.." + "Audit File rollover happens every " + rolloverPeriod );
+ }
+
+ executorService.scheduleAtFixedRate(new
AuditFilePeriodicRollOverTask(), 0, periodicRollOverCheckTimeinSec,
TimeUnit.SECONDS);
+ }
+
+ class AuditFilePeriodicRollOverTaskThreadFactory implements ThreadFactory {
+ //Threadfactory to create a daemon Thread.
+ public Thread newThread(Runnable r) {
+ Thread t = new Thread(r, "AuditFilePeriodicRollOverTask");
+ t.setDaemon(true);
+ return t;
+ }
+ }
+
+ private class AuditFilePeriodicRollOverTask implements Runnable {
+ public void run() {
+ if (logger.isDebugEnabled()) {
+ logger.debug("==> AuditFilePeriodicRollOverTask.run()");
+ }
+ try {
+ closeFileIfNeeded();
+ } catch (Exception excp) {
+ logger.error("AuditFilePeriodicRollOverTask Failed.
Aborting..", excp);
+ }
+ if (logger.isDebugEnabled()) {
+ logger.debug("<== AuditFilePeriodicRollOverTask.run()");
+ }
+ }
+ }
}
\ No newline at end of file
