RANGER-397 Incorporate review comments Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/8469e643 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/8469e643 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/8469e643
Branch: refs/heads/ranger-0.5 Commit: 8469e643da3a81312aff58dc74673ed48e440f95 Parents: c526417 Author: Don Bosco Durai <[email protected]> Authored: Mon Jun 1 18:02:42 2015 -0700 Committer: Don Bosco Durai <[email protected]> Committed: Mon Jun 1 18:02:42 2015 -0700 ---------------------------------------------------------------------- .../audit/destination/DBAuditDestination.java | 3 ++- .../audit/destination/FileAuditDestination.java | 3 ++- .../audit/destination/HDFSAuditDestination.java | 7 ++++-- .../destination/Log4JAuditDestination.java | 15 ++++++++--- .../audit/destination/SolrAuditDestination.java | 8 +++++- .../ranger/audit/provider/BaseAuditHandler.java | 14 +++++++++-- .../audit/provider/MultiDestAuditProvider.java | 21 +++++++++++++--- .../ranger/audit/queue/AuditBatchQueue.java | 5 ++-- .../apache/ranger/audit/queue/AuditQueue.java | 26 +++++++++++++++----- 9 files changed, 79 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java index 6825637..3d31c06 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java @@ -78,7 +78,7 @@ public class DBAuditDestination extends AuditDestination { @Override public boolean log(Collection<AuditEventBase> events) { boolean retValue = false; - logStatusIfRequired(true); + logStatusIfRequired(); addTotalCount(events.size()); if (beginTransaction()) { @@ -210,6 +210,7 @@ public class DBAuditDestination extends AuditDestination { entityManagerFactory = null; daoManager = null; } + logStatus(); } private EntityManager getEntityManager() { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java index 77c8f61..c6cd8b2 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java @@ -106,7 +106,7 @@ public class FileAuditDestination extends AuditDestination { @Override synchronized public boolean logJSON(Collection<String> events) { - logStatusIfRequired(true); + logStatusIfRequired(); addTotalCount(events.size()); if (isStopped) { @@ -183,6 +183,7 @@ public class FileAuditDestination extends AuditDestination { } logWriter = null; } + logStatus(); } // Helper methods in this class http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java index b63fa40..e79e42d 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java @@ -105,7 +105,7 @@ public class HDFSAuditDestination extends AuditDestination { @Override synchronized public boolean logJSON(Collection<String> events) { - logStatusIfRequired(true); + logStatusIfRequired(); addTotalCount(events.size()); if (!initDone) { @@ -142,8 +142,9 @@ public class HDFSAuditDestination extends AuditDestination { @Override public boolean log(Collection<AuditEventBase> events) { if (isStopped) { - logStatusIfRequired(true); + logStatusIfRequired(); addTotalCount(events.size()); + addDeferredCount(events.size()); logError("log() called after stop was requested. name=" + getName()); return false; } @@ -153,6 +154,7 @@ public class HDFSAuditDestination extends AuditDestination { jsonList.add(MiscUtil.stringify(event)); } catch (Throwable t) { logger.error("Error converting to JSON. event=" + event); + addTotalCount(1); addFailedCount(1); logFailedEvent(event); } @@ -185,6 +187,7 @@ public class HDFSAuditDestination extends AuditDestination { } logWriter = null; } + logStatus(); } // Helper methods in this class http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java index b57cd86..9521a4a 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java @@ -59,10 +59,17 @@ public class Log4JAuditDestination extends AuditDestination { + ", loggerName=" + loggerName); } + + @Override + public void stop() { + super.stop(); + logStatus(); + } + @Override public boolean log(AuditEventBase event) { if (!auditLogger.isInfoEnabled()) { - logStatusIfRequired(true); + logStatusIfRequired(); addTotalCount(1); return true; } @@ -77,7 +84,7 @@ public class Log4JAuditDestination extends AuditDestination { @Override public boolean log(Collection<AuditEventBase> events) { if (!auditLogger.isInfoEnabled()) { - logStatusIfRequired(true); + logStatusIfRequired(); addTotalCount(events.size()); return true; } @@ -90,7 +97,7 @@ public class Log4JAuditDestination extends AuditDestination { @Override public boolean logJSON(String event) { - logStatusIfRequired(true); + logStatusIfRequired(); addTotalCount(1); if (!auditLogger.isInfoEnabled()) { return true; @@ -106,7 +113,7 @@ public class Log4JAuditDestination extends AuditDestination { @Override public boolean logJSON(Collection<String> events) { if (!auditLogger.isInfoEnabled()) { - logStatusIfRequired(true); + logStatusIfRequired(); addTotalCount(events.size()); return true; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java index b43e573..213e4b2 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java @@ -55,6 +55,12 @@ public class SolrAuditDestination extends AuditDestination { super.init(props, propPrefix); connect(); } + + @Override + public void stop() { + super.stop(); + logStatus(); + } synchronized void connect() { if (solrClient == null) { @@ -103,7 +109,7 @@ public class SolrAuditDestination extends AuditDestination { @Override public boolean log(Collection<AuditEventBase> events) { try { - logStatusIfRequired(true); + logStatusIfRequired(); addTotalCount(events.size()); if (solrClient == null) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java index ef85743..3859a7e 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java @@ -234,9 +234,17 @@ public abstract class BaseAuditHandler implements AuditHandler { return lastDeferredCount; } - public void logStatusIfRequired(boolean ifRequired) { + public void logStatusIfRequired() { long currTime = System.currentTimeMillis(); - if (!ifRequired || (currTime - lastStatusLogTime) > statusLogIntervalMS) { + if ((currTime - lastStatusLogTime) > statusLogIntervalMS) { + logStatus(); + } + } + + public void logStatus() { + try { + long currTime = System.currentTimeMillis(); + long diffTime = currTime - lastStatusLogTime; lastStatusLogTime = currTime; @@ -278,6 +286,8 @@ public abstract class BaseAuditHandler implements AuditHandler { + (totalDeferredCount > 0 ? (", totalDeferredCount=" + totalDeferredCount) : ""); LOG.info(msg); + } catch (Throwable t) { + LOG.error("Error while printing stats. auditProvider=" + getName()); } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java index 9a5ff13..26108ca 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java @@ -33,13 +33,16 @@ public class MultiDestAuditProvider extends BaseAuditHandler { protected List<AuditHandler> mProviders = new ArrayList<AuditHandler>(); static final String DEFAULT_NAME = "multi_dest"; - + public MultiDestAuditProvider() { LOG.info("MultiDestAuditProvider: creating.."); setName(DEFAULT_NAME); } public MultiDestAuditProvider(AuditHandler provider) { + LOG.info("MultiDestAuditProvider(): provider=" + + (provider == null ? null : provider.getName())); + setName(DEFAULT_NAME); addAuditProvider(provider); } @@ -59,12 +62,22 @@ public class MultiDestAuditProvider extends BaseAuditHandler { } } - + @Override + public void setParentPath(String parentPath) { + super.setParentPath(parentPath); + for (AuditHandler provider : mProviders) { + if (provider instanceof BaseAuditHandler) { + BaseAuditHandler baseAuditHander = (BaseAuditHandler) provider; + baseAuditHander.setParentPath(getName()); + } + } + } + @Override public void setName(String name) { super.setName(name); for (AuditHandler provider : mProviders) { - if( provider instanceof BaseAuditHandler) { + if (provider instanceof BaseAuditHandler) { BaseAuditHandler baseAuditHander = (BaseAuditHandler) provider; baseAuditHander.setParentPath(getName()); } @@ -77,7 +90,7 @@ public class MultiDestAuditProvider extends BaseAuditHandler { + provider.getClass().getCanonicalName() + ")"); mProviders.add(provider); - if( provider instanceof BaseAuditHandler) { + if (provider instanceof BaseAuditHandler) { BaseAuditHandler baseAuditHander = (BaseAuditHandler) provider; baseAuditHander.setParentPath(getName()); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java index 4278731..2361bbf 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java @@ -39,7 +39,7 @@ public class AuditBatchQueue extends AuditQueue implements Runnable { Thread consumerThread = null; static int threadCount = 0; - static final String DEFAULT_NAME = "summary"; + static final String DEFAULT_NAME = "batch"; public AuditBatchQueue(AuditHandler consumer) { super(consumer); @@ -209,7 +209,7 @@ public class AuditBatchQueue extends AuditQueue implements Runnable { long lastDispatchTime = System.currentTimeMillis(); boolean isDestActive = true; while (true) { - logStatusIfRequired(true); + logStatusIfRequired(); // Time to next dispatch long nextDispatchDuration = lastDispatchTime @@ -347,6 +347,7 @@ public class AuditBatchQueue extends AuditQueue implements Runnable { } catch (Throwable t) { logger.error("Error while calling stop on consumer.", t); } + logStatus(); logger.info("Exiting consumerThread.run() method. name=" + getName()); } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java index 50a74b5..057f192 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java @@ -33,8 +33,9 @@ public abstract class AuditQueue extends BaseAuditHandler { public static final int AUDIT_MAX_QUEUE_SIZE_DEFAULT = 1024 * 1024; public static final int AUDIT_BATCH_INTERVAL_DEFAULT_MS = 1000; public static final int AUDIT_BATCH_SIZE_DEFAULT = 1000; - - //This is the max time the consumer thread will wait before exiting the loop + + // This is the max time the consumer thread will wait before exiting the + // loop public static final int AUDIT_CONSUMER_THREAD_WAIT_MS = 5000; private int maxQueueSize = AUDIT_MAX_QUEUE_SIZE_DEFAULT; @@ -60,14 +61,18 @@ public abstract class AuditQueue extends BaseAuditHandler { protected int fileSpoolMaxWaitTime = 5 * 60 * 1000; // Default 5 minutes protected int fileSpoolDrainThresholdPercent = 80; - //This is set when the first time stop is called. + // This is set when the first time stop is called. protected long stopTime = 0; - + /** * @param consumer */ public AuditQueue(AuditHandler consumer) { this.consumer = consumer; + if (consumer instanceof BaseAuditHandler) { + BaseAuditHandler baseAuditHander = (BaseAuditHandler) consumer; + baseAuditHander.setParentPath(getName()); + } } @Override @@ -111,9 +116,18 @@ public abstract class AuditQueue extends BaseAuditHandler { } @Override + public void setParentPath(String parentPath) { + super.setParentPath(parentPath); + if (consumer != null && consumer instanceof BaseAuditHandler) { + BaseAuditHandler base = (BaseAuditHandler) consumer; + base.setParentPath(getName()); + } + } + + @Override public void setName(String name) { super.setName(name); - if( consumer != null && consumer instanceof BaseAuditHandler) { + if (consumer != null && consumer instanceof BaseAuditHandler) { BaseAuditHandler base = (BaseAuditHandler) consumer; base.setParentPath(getName()); } @@ -126,7 +140,7 @@ public abstract class AuditQueue extends BaseAuditHandler { public boolean isDrainMaxTimeElapsed() { return (stopTime - System.currentTimeMillis()) > AUDIT_CONSUMER_THREAD_WAIT_MS; } - + public boolean isDrain() { return isDrain; }
