Repository: incubator-ranger Updated Branches: refs/heads/master c510b449d -> e0e5105c9
RANGER-397 Port log4j audit provider to V3 implementation Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/e92b2d38 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/e92b2d38 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/e92b2d38 Branch: refs/heads/master Commit: e92b2d38fa513ae9b92644f9cd72e61818dc41b6 Parents: c510b44 Author: Don Bosco Durai <[email protected]> Authored: Sun May 31 23:47:26 2015 -0700 Committer: Don Bosco Durai <[email protected]> Committed: Sun May 31 23:47:26 2015 -0700 ---------------------------------------------------------------------- .../audit/destination/DBAuditDestination.java | 42 +++--- .../audit/destination/FileAuditDestination.java | 10 ++ .../audit/destination/HDFSAuditDestination.java | 11 ++ .../destination/Log4JAuditDestination.java | 120 ++++++++++++++++ .../audit/destination/SolrAuditDestination.java | 11 +- .../audit/provider/AuditProviderFactory.java | 8 +- .../ranger/audit/provider/BaseAuditHandler.java | 138 ++++++++++++++++++- .../audit/provider/Log4jAuditProvider.java | 2 +- .../audit/provider/MultiDestAuditProvider.java | 20 ++- .../ranger/audit/queue/AuditBatchQueue.java | 9 ++ .../ranger/audit/queue/AuditFileSpool.java | 4 + .../apache/ranger/audit/queue/AuditQueue.java | 30 ++++ .../admin/client/RangerAdminRESTClient.java | 7 +- .../plugin/audit/RangerDefaultAuditHandler.java | 5 +- 14 files changed, 384 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java index 8cece4e..6825637 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java @@ -78,25 +78,32 @@ public class DBAuditDestination extends AuditDestination { @Override public boolean log(Collection<AuditEventBase> events) { boolean retValue = false; - - if (!beginTransaction()) { - return false; - } - boolean isFailed = false; - for (AuditEventBase event : events) { - try { - event.persist(daoManager); - } catch (Throwable t) { - logger.error("Error persisting data. event=" + event, t); - isFailed = true; - break; + logStatusIfRequired(true); + addTotalCount(events.size()); + + if (beginTransaction()) { + boolean isFailed = false; + for (AuditEventBase event : events) { + try { + event.persist(daoManager); + } catch (Throwable t) { + logger.error("Error persisting data. event=" + event, t); + isFailed = true; + break; + } + } + if (isFailed) { + retValue = false; + rollbackTransaction(); + } else { + retValue = commitTransaction(); } } - if (isFailed) { - retValue = false; - rollbackTransaction(); + + if (retValue) { + addSuccessCount(events.size()); } else { - retValue = commitTransaction(); + addDeferredCount(events.size()); } return retValue; } @@ -179,6 +186,9 @@ public class DBAuditDestination extends AuditDestination { if (daoManager.getEntityManager() == null) { logger.error("Error connecting audit database. EntityManager is null. dbURL=" + jdbcURL + ", dbUser=" + dbUser); + } else { + logger.info("Connected to audit database. dbURL=" + jdbcURL + + ", dbUser=" + dbUser); } } catch (Throwable t) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java index a132cdf..854592d 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java @@ -106,8 +106,11 @@ public class FileAuditDestination extends AuditDestination { @Override synchronized public boolean logJSON(Collection<String> events) { + logStatusIfRequired(true); if (isStopped) { logError("log() called after stop was requested. name=" + getName()); + addTotalCount(events.size()); + addDeferredCount(events.size()); return false; } @@ -118,9 +121,11 @@ public class FileAuditDestination extends AuditDestination { } out.flush(); } catch (Throwable t) { + addDeferredCount(events.size()); logError("Error writing to log file.", t); return false; } + addSuccessCount(events.size()); return true; } @@ -133,6 +138,8 @@ public class FileAuditDestination extends AuditDestination { @Override public boolean log(Collection<AuditEventBase> events) { if (isStopped) { + addTotalCount(events.size()); + addDeferredCount(events.size()); logError("log() called after stop was requested. name=" + getName()); return false; } @@ -141,6 +148,9 @@ public class FileAuditDestination extends AuditDestination { try { jsonList.add(MiscUtil.stringify(event)); } catch (Throwable t) { + addTotalCount(1); + addFailedCount(1); + logFailedEvent(event); logger.error("Error converting to JSON. event=" + event); } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java index 67382a9..b63fa40 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java @@ -105,10 +105,15 @@ public class HDFSAuditDestination extends AuditDestination { @Override synchronized public boolean logJSON(Collection<String> events) { + logStatusIfRequired(true); + addTotalCount(events.size()); + if (!initDone) { + addDeferredCount(events.size()); return false; } if (isStopped) { + addDeferredCount(events.size()); logError("log() called after stop was requested. name=" + getName()); return false; } @@ -120,9 +125,11 @@ public class HDFSAuditDestination extends AuditDestination { } out.flush(); } catch (Throwable t) { + addDeferredCount(events.size()); logError("Error writing to log file.", t); return false; } + addSuccessCount(events.size()); return true; } @@ -135,6 +142,8 @@ public class HDFSAuditDestination extends AuditDestination { @Override public boolean log(Collection<AuditEventBase> events) { if (isStopped) { + logStatusIfRequired(true); + addTotalCount(events.size()); logError("log() called after stop was requested. name=" + getName()); return false; } @@ -144,6 +153,8 @@ public class HDFSAuditDestination extends AuditDestination { jsonList.add(MiscUtil.stringify(event)); } catch (Throwable t) { logger.error("Error converting to JSON. event=" + event); + addFailedCount(1); + logFailedEvent(event); } } return logJSON(jsonList); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java new file mode 100644 index 0000000..b57cd86 --- /dev/null +++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.destination; + +import java.util.Collection; +import java.util.Properties; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.audit.model.AuditEventBase; +import org.apache.ranger.audit.provider.MiscUtil; + +public class Log4JAuditDestination extends AuditDestination { + private static final Log logger = LogFactory + .getLog(FileAuditDestination.class); + + private static Log auditLogger = null; + + public static final String PROP_LOG4J_LOGGER = "logger"; + public static final String DEFAULT_LOGGER_PREFIX = "ranger.audit"; + private String loggerName = null; + + public Log4JAuditDestination() { + logger.info("Log4JAuditDestination() called."); + + } + + @Override + public void init(Properties prop, String propPrefix) { + super.init(prop, propPrefix); + loggerName = MiscUtil.getStringProperty(props, propPrefix + "." + + PROP_LOG4J_LOGGER); + if (loggerName == null || loggerName.isEmpty()) { + loggerName = DEFAULT_LOGGER_PREFIX + "." + getName(); + logger.info("Logger property " + propPrefix + "." + + PROP_LOG4J_LOGGER + " was not set. Constructing default=" + + loggerName); + } + logger.info("Logger name for " + getName() + " is " + loggerName); + auditLogger = LogFactory.getLog(loggerName); + logger.info("Done initializing logger for audit. name=" + getName() + + ", loggerName=" + loggerName); + } + + @Override + public boolean log(AuditEventBase event) { + if (!auditLogger.isInfoEnabled()) { + logStatusIfRequired(true); + addTotalCount(1); + return true; + } + + if (event != null) { + String eventStr = MiscUtil.stringify(event); + logJSON(eventStr); + } + return true; + } + + @Override + public boolean log(Collection<AuditEventBase> events) { + if (!auditLogger.isInfoEnabled()) { + logStatusIfRequired(true); + addTotalCount(events.size()); + return true; + } + + for (AuditEventBase event : events) { + log(event); + } + return true; + } + + @Override + public boolean logJSON(String event) { + logStatusIfRequired(true); + addTotalCount(1); + if (!auditLogger.isInfoEnabled()) { + return true; + } + + if (event != null) { + auditLogger.info(event); + addSuccessCount(1); + } + return true; + } + + @Override + public boolean logJSON(Collection<String> events) { + if (!auditLogger.isInfoEnabled()) { + logStatusIfRequired(true); + addTotalCount(events.size()); + return true; + } + + for (String event : events) { + logJSON(event); + } + return false; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java index ac522cd..b43e573 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java @@ -103,10 +103,14 @@ public class SolrAuditDestination extends AuditDestination { @Override public boolean log(Collection<AuditEventBase> events) { try { + logStatusIfRequired(true); + addTotalCount(events.size()); + if (solrClient == null) { connect(); if (solrClient == null) { - // Solr is still not initialized. So need to throw error + // Solr is still not initialized. So need return error + addDeferredCount(events.size()); return false; } } @@ -121,12 +125,17 @@ public class SolrAuditDestination extends AuditDestination { try { UpdateResponse response = solrClient.add(docs); if (response.getStatus() != 0) { + addFailedCount(events.size()); logFailedEvent(events, response.toString()); + } else { + addSuccessCount(events.size()); } } catch (SolrException ex) { + addFailedCount(events.size()); logFailedEvent(events, ex); } } catch (Throwable t) { + addDeferredCount(events.size()); logError("Error sending message to Solr", t); return false; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java index c3a05ce..f02ba62 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java @@ -27,6 +27,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.ranger.audit.destination.DBAuditDestination; import org.apache.ranger.audit.destination.FileAuditDestination; import org.apache.ranger.audit.destination.HDFSAuditDestination; +import org.apache.ranger.audit.destination.Log4JAuditDestination; import org.apache.ranger.audit.destination.SolrAuditDestination; import org.apache.ranger.audit.provider.hdfs.HdfsAuditProvider; import org.apache.ranger.audit.provider.kafka.KafkaAuditProvider; @@ -62,6 +63,7 @@ public class AuditProviderFactory { private static AuditProviderFactory sFactory; private AuditHandler mProvider = null; + private String componentAppType = ""; private boolean mInitDone = false; private AuditProviderFactory() { @@ -105,7 +107,7 @@ public class AuditProviderFactory { // return; } mInitDone = true; - + componentAppType = appType; MiscUtil.setApplicationType(appType); boolean isEnabled = MiscUtil.getBooleanProperty(props, @@ -240,7 +242,7 @@ public class AuditProviderFactory { AuditAsyncQueue asyncQueue = new AuditAsyncQueue(consumer); propPrefix = BaseAuditHandler.PROP_DEFAULT_PREFIX + "." + "async"; asyncQueue.init(props, propPrefix); - + asyncQueue.setParentPath(componentAppType); mProvider = asyncQueue; LOG.info("Starting audit queue " + mProvider.getName()); mProvider.start(); @@ -418,7 +420,7 @@ public class AuditProviderFactory { } else if (providerName.equals("db")) { provider = new DBAuditDestination(); } else if (providerName.equals("log4j")) { - provider = new Log4jAuditProvider(); + provider = new Log4JAuditDestination(); } else if (providerName.equals("batch")) { provider = new AuditBatchQueue(consumer); } else if (providerName.equals("async")) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java index 09335c7..ef85743 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java @@ -35,8 +35,7 @@ public abstract class BaseAuditHandler implements AuditHandler { private static final Log LOG = LogFactory.getLog(BaseAuditHandler.class); static final String AUDIT_LOG_FAILURE_REPORT_MIN_INTERVAL_PROP = "xasecure.audit.log.failure.report.min.interval.ms"; - protected static final String AUDIT_DB_CREDENTIAL_PROVIDER_FILE = "xasecure.audit.credential.provider.file"; - + protected static final String AUDIT_DB_CREDENTIAL_PROVIDER_FILE = "xasecure.audit.credential.provider.file"; private int mLogFailureReportMinIntervalInMs = 60 * 1000; @@ -52,6 +51,7 @@ public abstract class BaseAuditHandler implements AuditHandler { protected String propPrefix = PROP_DEFAULT_PREFIX; protected String providerName = null; + protected String parentPath = null; protected int failedRetryTimes = 3; protected int failedRetrySleep = 3 * 1000; @@ -59,6 +59,21 @@ public abstract class BaseAuditHandler implements AuditHandler { int errorLogIntervalMS = 30 * 1000; // Every 30 seconds long lastErrorLogMS = 0; + long totalCount = 0; + long totalSuccessCount = 0; + long totalFailedCount = 0; + long totalStashedCount = 0; + long totalDeferredCount = 0; + + long lastIntervalCount = 0; + long lastIntervalSuccessCount = 0; + long lastIntervalFailedCount = 0; + long lastStashedCount = 0; + long lastDeferredCount = 0; + + long lastStatusLogTime = System.currentTimeMillis(); + long statusLogIntervalMS = 1 * 60 * 1000; + protected Properties props = null; @Override @@ -81,14 +96,14 @@ public abstract class BaseAuditHandler implements AuditHandler { String name = MiscUtil.getStringProperty(props, basePropertyName + "." + PROP_NAME); if (name != null && !name.isEmpty()) { - providerName = name; + setName(name); } if (providerName == null) { - providerName = finalToken; + setName(finalToken); LOG.info("Using providerName from property prefix. providerName=" - + providerName); + + getName()); } - LOG.info("providerName=" + providerName); + LOG.info("providerName=" + getName()); try { new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").create(); @@ -146,15 +161,126 @@ public abstract class BaseAuditHandler implements AuditHandler { return log(eventList); } + public String getParentPath() { + return parentPath; + } + + public void setParentPath(String parentPath) { + this.parentPath = parentPath; + } + public void setName(String name) { providerName = name; } @Override public String getName() { + if (parentPath != null) { + return parentPath + "." + providerName; + } return providerName; } + public long addTotalCount(int count) { + totalCount += count; + return totalCount; + } + + public long addSuccessCount(int count) { + totalSuccessCount += count; + return totalSuccessCount; + } + + public long addFailedCount(int count) { + totalFailedCount += count; + return totalFailedCount; + } + + public long addStashedCount(int count) { + totalStashedCount += count; + return totalStashedCount; + } + + public long addDeferredCount(int count) { + totalDeferredCount += count; + return totalDeferredCount; + } + + public long getTotalCount() { + return totalCount; + } + + public long getTotalSuccessCount() { + return totalSuccessCount; + } + + public long getTotalFailedCount() { + return totalFailedCount; + } + + public long getTotalStashedCount() { + return totalStashedCount; + } + + public long getLastStashedCount() { + return lastStashedCount; + } + + public long getTotalDeferredCount() { + return totalDeferredCount; + } + + public long getLastDeferredCount() { + return lastDeferredCount; + } + + public void logStatusIfRequired(boolean ifRequired) { + long currTime = System.currentTimeMillis(); + if (!ifRequired || (currTime - lastStatusLogTime) > statusLogIntervalMS) { + long diffTime = currTime - lastStatusLogTime; + lastStatusLogTime = currTime; + + long diffCount = totalCount - lastIntervalCount; + if (diffCount == 0) { + return; + } + long diffSuccess = totalSuccessCount - lastIntervalSuccessCount; + long diffFailed = totalFailedCount - lastIntervalFailedCount; + long diffStashed = totalStashedCount - lastStashedCount; + long diffDeferred = totalDeferredCount - lastDeferredCount; + + lastIntervalCount = totalCount; + lastIntervalSuccessCount = totalSuccessCount; + lastIntervalFailedCount = totalFailedCount; + lastStashedCount = totalStashedCount; + lastDeferredCount = totalDeferredCount; + + String msg = "Audit Status Log: name=" + + getName() + + ", interval=" + + formatIntervalForLog(diffTime) + + ", events=" + + diffCount + + (diffSuccess > 0 ? (", succcessCount=" + diffSuccess) + : "") + + (diffFailed > 0 ? (", failedCount=" + diffFailed) : "") + + (diffStashed > 0 ? (", stashedCount=" + diffStashed) : "") + + (diffDeferred > 0 ? (", deferredCount=" + diffDeferred) + : "") + + ", totalEvents=" + + totalCount + + (totalSuccessCount > 0 ? (", totalSuccessCount=" + totalSuccessCount) + : "") + + (totalFailedCount > 0 ? (", totalFailedCount=" + totalFailedCount) + : "") + + (totalStashedCount > 0 ? (", totalStashedCount=" + totalStashedCount) + : "") + + (totalDeferredCount > 0 ? (", totalDeferredCount=" + totalDeferredCount) + : ""); + LOG.info(msg); + } + } + public void logError(String msg) { long currTimeMS = System.currentTimeMillis(); if (currTimeMS - lastErrorLogMS > errorLogIntervalMS) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-audit/src/main/java/org/apache/ranger/audit/provider/Log4jAuditProvider.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/Log4jAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/Log4jAuditProvider.java index 0402de2..3beab69 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/Log4jAuditProvider.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/Log4jAuditProvider.java @@ -81,7 +81,7 @@ public class Log4jAuditProvider extends AuditDestination { for (String event : events) { logJSON(event); } - return false; + return true; } @Override http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java index 4c1593a..9a5ff13 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java @@ -32,9 +32,11 @@ public class MultiDestAuditProvider extends BaseAuditHandler { .getLog(MultiDestAuditProvider.class); protected List<AuditHandler> mProviders = new ArrayList<AuditHandler>(); - + static final String DEFAULT_NAME = "multi_dest"; + public MultiDestAuditProvider() { LOG.info("MultiDestAuditProvider: creating.."); + setName(DEFAULT_NAME); } public MultiDestAuditProvider(AuditHandler provider) { @@ -57,12 +59,28 @@ public class MultiDestAuditProvider extends BaseAuditHandler { } } + + @Override + public void setName(String name) { + super.setName(name); + for (AuditHandler provider : mProviders) { + if( provider instanceof BaseAuditHandler) { + BaseAuditHandler baseAuditHander = (BaseAuditHandler) provider; + baseAuditHander.setParentPath(getName()); + } + } + } + public void addAuditProvider(AuditHandler provider) { if (provider != null) { LOG.info("MultiDestAuditProvider.addAuditProvider(providerType=" + provider.getClass().getCanonicalName() + ")"); mProviders.add(provider); + if( provider instanceof BaseAuditHandler) { + BaseAuditHandler baseAuditHander = (BaseAuditHandler) provider; + baseAuditHander.setParentPath(getName()); + } } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java index 80d7853..4278731 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java @@ -39,9 +39,11 @@ public class AuditBatchQueue extends AuditQueue implements Runnable { Thread consumerThread = null; static int threadCount = 0; + static final String DEFAULT_NAME = "summary"; public AuditBatchQueue(AuditHandler consumer) { super(consumer); + setName(DEFAULT_NAME); } /* @@ -207,6 +209,8 @@ public class AuditBatchQueue extends AuditQueue implements Runnable { long lastDispatchTime = System.currentTimeMillis(); boolean isDestActive = true; while (true) { + logStatusIfRequired(true); + // Time to next dispatch long nextDispatchDuration = lastDispatchTime - System.currentTimeMillis() + getMaxBatchInterval(); @@ -269,6 +273,7 @@ public class AuditBatchQueue extends AuditQueue implements Runnable { logger.error("Caught error during processing request.", t); } + addTotalCount(localBatchBuffer.size()); if (localBatchBuffer.size() > 0 && isToSpool) { // Let spool to the file directly if (isDestActive) { @@ -279,6 +284,7 @@ public class AuditBatchQueue extends AuditQueue implements Runnable { // Just before stashing lastDispatchTime = System.currentTimeMillis(); fileSpooler.stashLogs(localBatchBuffer); + addStashedCount(localBatchBuffer.size()); localBatchBuffer.clear(); } else if (localBatchBuffer.size() > 0 && (isDrain() @@ -297,12 +303,15 @@ public class AuditBatchQueue extends AuditQueue implements Runnable { // Transient error. Stash and move on fileSpooler.stashLogs(localBatchBuffer); isDestActive = false; + addStashedCount(localBatchBuffer.size()); } else { // We need to drop this event + addFailedCount(localBatchBuffer.size()); logFailedEvent(localBatchBuffer); } } else { isDestActive = true; + addSuccessCount(localBatchBuffer.size()); } localBatchBuffer.clear(); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileSpool.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileSpool.java b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileSpool.java index 1b9a921..9c10c54 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileSpool.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileSpool.java @@ -842,6 +842,10 @@ public class AuditFileSpool implements Runnable { currentConsumerIndexRecord = null; closeFileIfNeeded(); } + } catch (InterruptedException e) { + logger.info( + "Caught exception in consumer thread. Shutdown might be in progress", + e); } catch (Throwable t) { logger.error("Exception in destination writing thread.", t); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java index e873459..7681b82 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java @@ -19,10 +19,12 @@ package org.apache.ranger.audit.queue; +import java.util.Collection; import java.util.Properties; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.ranger.audit.model.AuditEventBase; import org.apache.ranger.audit.provider.AuditHandler; import org.apache.ranger.audit.provider.BaseAuditHandler; import org.apache.ranger.audit.provider.MiscUtil; @@ -110,6 +112,34 @@ public abstract class AuditQueue extends BaseAuditHandler { } + + @Override + public boolean log(Collection<AuditEventBase> events) { + // TODO Auto-generated method stub + return false; + } + + @Override + public void start() { + // TODO Auto-generated method stub + + } + + @Override + public void stop() { + // TODO Auto-generated method stub + + } + + @Override + public void setName(String name) { + super.setName(name); + if( consumer != null && consumer instanceof BaseAuditHandler) { + BaseAuditHandler base = (BaseAuditHandler) consumer; + base.setParentPath(getName()); + } + } + public AuditHandler getConsumer() { return consumer; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java index 9d103bb..51082bd 100644 --- a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java +++ b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java @@ -76,7 +76,8 @@ public class RangerAdminRESTClient implements RangerAdminClient { // no change } else { RESTResponse resp = RESTResponse.fromClientResponse(response); - + LOG.error("Error getting policies. request=" + webResource.toString() + + ", response=" + response.toString() + ", serviceName=" + serviceName); throw new Exception(resp.getMessage()); } @@ -106,7 +107,7 @@ public class RangerAdminRESTClient implements RangerAdminClient { throw new Exception("HTTP " + response.getStatus()); } else if(response == null) { - throw new Exception("unknown error"); + throw new Exception("unknown error during grantAccess. serviceName=" + serviceName); } if(LOG.isDebugEnabled()) { @@ -133,7 +134,7 @@ public class RangerAdminRESTClient implements RangerAdminClient { throw new Exception("HTTP " + response.getStatus()); } else if(response == null) { - throw new Exception("unknown error"); + throw new Exception("unknown error. revokeAccess(). serviceName=" + serviceName); } if(LOG.isDebugEnabled()) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java index 0d38224..253a180 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java @@ -37,7 +37,7 @@ import org.apache.ranger.plugin.policyengine.RangerAccessResultProcessor; public class RangerDefaultAuditHandler implements RangerAccessResultProcessor { private static final Log LOG = LogFactory.getLog(RangerDefaultAuditHandler.class); - + static long sequenceNumber = 0; public RangerDefaultAuditHandler() { } @@ -176,7 +176,8 @@ public class RangerDefaultAuditHandler implements RangerAccessResultProcessor { if (auditEvent.getEventId() == null || auditEvent.getEventId().isEmpty()) { auditEvent.setEventId(MiscUtil.generateUniqueId()); - } + } + auditEvent.setSeqNum(sequenceNumber++); } public void logAuthzAudits(Collection<AuthzAuditEvent> auditEvents) {
