RANGER-615 Audit to db: Truncate all string values of audit record so that writing of audit does not fail
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/33f84070 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/33f84070 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/33f84070 Branch: refs/heads/HDP-2.3.2-groupid Commit: 33f840704962d7ed5d5e26c63c8f7247d5faeaf9 Parents: eec5ac4 Author: Alok Lal <[email protected]> Authored: Thu Aug 13 08:48:14 2015 -0700 Committer: Alok Lal <[email protected]> Committed: Fri Aug 21 10:15:11 2015 -0700 ---------------------------------------------------------------------- .../audit/destination/DBAuditDestination.java | 5 +- .../audit/entity/AuthzAuditEventDbObj.java | 143 +++++++++++++++++-- .../ranger/audit/model/AuditEventBase.java | 14 +- .../ranger/audit/model/AuthzAuditEvent.java | 12 +- .../ranger/audit/provider/DbAuditProvider.java | 4 + 5 files changed, 144 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/33f84070/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java index 3d31c06..376e724 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java @@ -33,6 +33,7 @@ import javax.persistence.EntityTransaction; import javax.persistence.Persistence; import org.apache.ranger.audit.dao.DaoManager; +import org.apache.ranger.audit.entity.AuthzAuditEventDbObj; import org.apache.ranger.audit.model.AuditEventBase; import org.apache.ranger.audit.provider.MiscUtil; @@ -63,9 +64,11 @@ public class DBAuditDestination extends AuditDestination { public void init(Properties props, String propPrefix) { logger.info("init() called"); super.init(props, propPrefix); - // Initial connect connect(); + + // initialize the database related classes + AuthzAuditEventDbObj.init(props); } /* http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/33f84070/agents-audit/src/main/java/org/apache/ranger/audit/entity/AuthzAuditEventDbObj.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/entity/AuthzAuditEventDbObj.java b/agents-audit/src/main/java/org/apache/ranger/audit/entity/AuthzAuditEventDbObj.java index 435393e..d52a60a 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/entity/AuthzAuditEventDbObj.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/entity/AuthzAuditEventDbObj.java @@ -21,6 +21,7 @@ import java.io.Serializable; import java.util.Date; +import java.util.Properties; import javax.persistence.Column; import javax.persistence.Entity; @@ -32,8 +33,11 @@ import javax.persistence.Temporal; import javax.persistence.TemporalType; import javax.persistence.SequenceGenerator; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.ranger.audit.model.EnumRepositoryType; import org.apache.ranger.audit.model.AuthzAuditEvent; +import org.apache.ranger.audit.provider.MiscUtil; /** * Entity implementation class for Entity: AuthzAuditEventDbObj @@ -42,8 +46,25 @@ import org.apache.ranger.audit.model.AuthzAuditEvent; @Entity @Table(name="xa_access_audit") public class AuthzAuditEventDbObj implements Serializable { + + private static final Log LOG = LogFactory.getLog(AuthzAuditEventDbObj.class); + private static final long serialVersionUID = 1L; + static int MaxValueLengthAccessType = 255; + static int MaxValueLengthAclEnforcer = 255; + static int MaxValueLengthAgentId = 255; + static int MaxValueLengthClientIp = 255; + static int MaxValueLengthClientType = 255; + static int MaxValueLengthRepoName = 255; + static int MaxValueLengthResultReason = 255; + static int MaxValueLengthSessionId = 255; + static int MaxValueLengthRequestUser = 255; + static int MaxValueLengthAction = 2000; + static int MaxValueLengthRequestData = 4000; + static int MaxValueLengthResourcePath = 4000; + static int MaxValueLengthResourceType = 255; + private long auditId; private int repositoryType; private String repositoryName; @@ -63,6 +84,60 @@ public class AuthzAuditEventDbObj implements Serializable { private String clientIP; private String requestData; + public static void init(Properties props) + { + LOG.info("AuthzAuditEventDbObj.init()"); + + final String AUDIT_DB_MAX_COLUMN_VALUE = "xasecure.audit.destination.db.max.column.length"; + MaxValueLengthAccessType = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "access_type", MaxValueLengthAccessType); + logMaxColumnValue("access_type", MaxValueLengthAccessType); + + MaxValueLengthAclEnforcer = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "acl_enforcer", MaxValueLengthAclEnforcer); + logMaxColumnValue("acl_enforcer", MaxValueLengthAclEnforcer); + + MaxValueLengthAction = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "action", MaxValueLengthAction); + logMaxColumnValue("action", MaxValueLengthAction); + + MaxValueLengthAgentId = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "agent_id", MaxValueLengthAgentId); + logMaxColumnValue("agent_id", MaxValueLengthAgentId); + + MaxValueLengthClientIp = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "client_id", MaxValueLengthClientIp); + logMaxColumnValue("client_id", MaxValueLengthClientIp); + + MaxValueLengthClientType = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "client_type", MaxValueLengthClientType); + logMaxColumnValue("client_type", MaxValueLengthClientType); + + MaxValueLengthRepoName = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "repo_name", MaxValueLengthRepoName); + logMaxColumnValue("repo_name", MaxValueLengthRepoName); + + MaxValueLengthResultReason = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "result_reason", MaxValueLengthResultReason); + logMaxColumnValue("result_reason", MaxValueLengthResultReason); + + MaxValueLengthSessionId = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "session_id", MaxValueLengthSessionId); + logMaxColumnValue("session_id", MaxValueLengthSessionId); + + MaxValueLengthRequestUser = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "request_user", MaxValueLengthRequestUser); + logMaxColumnValue("request_user", MaxValueLengthRequestUser); + + MaxValueLengthRequestData = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "request_data", MaxValueLengthRequestData); + logMaxColumnValue("request_data", MaxValueLengthRequestData); + + MaxValueLengthResourcePath = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "resource_path", MaxValueLengthResourcePath); + logMaxColumnValue("resource_path", MaxValueLengthResourcePath); + + MaxValueLengthResourceType = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." + "resource_type", MaxValueLengthResourceType); + logMaxColumnValue("resource_type", MaxValueLengthResourceType); + } + + public static void logMaxColumnValue(String columnName, int configuredMaxValueLength) { + LOG.info("Setting max column value for column[" + columnName + "] to [" + configuredMaxValueLength + "]."); + if (configuredMaxValueLength == 0) { + LOG.info("Max length of column[" + columnName + "] was 0! Column will NOT be emitted in the audit."); + } else if (configuredMaxValueLength < 0) { + LOG.info("Max length of column[" + columnName + "] was less than 0! Column value will never be truncated."); + } + } + public AuthzAuditEventDbObj() { super(); @@ -113,7 +188,7 @@ public class AuthzAuditEventDbObj implements Serializable { @Column(name = "repo_name") public String getRepositoryName() { - return this.repositoryName; + return truncate(this.repositoryName, MaxValueLengthRepoName, "repo_name"); } public void setRepositoryName(String repositoryName) { @@ -122,7 +197,7 @@ public class AuthzAuditEventDbObj implements Serializable { @Column(name = "request_user") public String getUser() { - return this.user; + return truncate(this.user, MaxValueLengthRequestUser, "request_user"); } public void setUser(String user) { @@ -141,7 +216,7 @@ public class AuthzAuditEventDbObj implements Serializable { @Column(name = "access_type") public String getAccessType() { - return this.accessType; + return truncate(this.accessType, MaxValueLengthAccessType, "access_type"); } public void setAccessType(String accessType) { @@ -150,7 +225,7 @@ public class AuthzAuditEventDbObj implements Serializable { @Column(name = "resource_path") public String getResourcePath() { - return this.resourcePath; + return truncate(this.resourcePath, MaxValueLengthResourcePath, "resource_path"); } public void setResourcePath(String resourcePath) { @@ -159,7 +234,7 @@ public class AuthzAuditEventDbObj implements Serializable { @Column(name = "resource_type") public String getResourceType() { - return this.resourceType; + return truncate(this.resourceType, MaxValueLengthResourceType, "resource_type"); } public void setResourceType(String resourceType) { @@ -168,7 +243,7 @@ public class AuthzAuditEventDbObj implements Serializable { @Column(name = "action") public String getAction() { - return this.action; + return truncate(this.action, MaxValueLengthAction, "action"); } public void setAction(String action) { @@ -186,7 +261,7 @@ public class AuthzAuditEventDbObj implements Serializable { @Column(name = "agent_id") public String getAgentId() { - return agentId; + return truncate(this.agentId, MaxValueLengthAgentId, "agent_id"); } public void setAgentId(String agentId) { @@ -204,7 +279,7 @@ public class AuthzAuditEventDbObj implements Serializable { @Column(name = "result_reason") public String getResultReason() { - return this.resultReason; + return truncate(this.resultReason, MaxValueLengthResultReason, "result_reason"); } public void setResultReason(String resultReason) { @@ -213,7 +288,7 @@ public class AuthzAuditEventDbObj implements Serializable { @Column(name = "acl_enforcer") public String getAclEnforcer() { - return this.aclEnforcer; + return truncate(this.aclEnforcer, MaxValueLengthAclEnforcer, "acl_enforcer"); } public void setAclEnforcer(String aclEnforcer) { @@ -222,7 +297,7 @@ public class AuthzAuditEventDbObj implements Serializable { @Column(name = "session_id") public String getSessionId() { - return this.sessionId; + return truncate(this.sessionId, MaxValueLengthSessionId, "session_id"); } public void setSessionId(String sessionId) { @@ -231,7 +306,7 @@ public class AuthzAuditEventDbObj implements Serializable { @Column(name = "client_type") public String getClientType() { - return this.clientType; + return truncate(this.clientType, MaxValueLengthClientType, "client_type"); } public void setClientType(String clientType) { @@ -240,7 +315,7 @@ public class AuthzAuditEventDbObj implements Serializable { @Column(name = "client_ip") public String getClientIP() { - return this.clientIP; + return truncate(this.clientIP, MaxValueLengthClientIp, "client_ip"); } public void setClientIP(String clientIP) { @@ -249,10 +324,52 @@ public class AuthzAuditEventDbObj implements Serializable { @Column(name = "request_data") public String getRequestData() { - return this.requestData; + return truncate(this.requestData, MaxValueLengthRequestData, "request_data"); } public void setRequestData(String requestData) { this.requestData = requestData; } + static final String TruncationMarker = "..."; + static final int TruncationMarkerLength = TruncationMarker.length(); + + protected String truncate(String value, int limit, String columnName) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("==> getTrunctedValue(%s, %d, %s)", value, limit, columnName)); + } + + String result = value; + if (value != null) { + if (limit < 0) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("Truncation is suppressed for column[%s]: old value [%s], new value[%s]", columnName, value, result)); + } + } else if (limit == 0) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("Column[%s] is to be excluded from audit: old value [%s], new value[%s]", columnName, value, result)); + } + result = null; + } else { + if (value.length() > limit) { + if (limit <= TruncationMarkerLength) { + // NOTE: If value is to be truncated to a size that is less than of equal to the Truncation Marker then we won't put the marker in!! + result = value.substring(0, limit); + } else { + StringBuilder sb = new StringBuilder(value.substring(0, limit - TruncationMarkerLength)); + sb.append(TruncationMarker); + result = sb.toString(); + } + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("Truncating value for column[%s] to [%d] characters: old value [%s], new value[%s]", columnName, limit, value, result)); + } + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("<== getTrunctedValue(%s, %d, %s): %s", value, limit, columnName, result)); + } + return result; + } + } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/33f84070/agents-audit/src/main/java/org/apache/ranger/audit/model/AuditEventBase.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/model/AuditEventBase.java b/agents-audit/src/main/java/org/apache/ranger/audit/model/AuditEventBase.java index 2c6a87f..2a07e94 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/model/AuditEventBase.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/model/AuditEventBase.java @@ -21,6 +21,8 @@ package org.apache.ranger.audit.model; import java.util.Date; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.ranger.audit.dao.DaoManager; public abstract class AuditEventBase { @@ -34,16 +36,4 @@ public abstract class AuditEventBase { public abstract Date getEventTime (); public abstract void setEventCount(long eventCount); public abstract void setEventDurationMS(long eventDurationMS); - - protected String trim(String str, int len) { - String ret = str; - if (str != null) { - if (str.length() > len) { - ret = str.substring(0, len); - } - } - return ret; - } - - } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/33f84070/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java b/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java index d648de3..2a8d792 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java @@ -190,8 +190,8 @@ public class AuthzAuditEvent extends AuditEventBase { } /** - * @param timeStamp - * the timeStamp to set + * @param eventTime + * the eventTime to set */ public void setEventTime(Date eventTime) { this.eventTime = eventTime; @@ -245,9 +245,7 @@ public class AuthzAuditEvent extends AuditEventBase { /** * @return the action */ - public String getAction() { - return trim(action, MAX_ACTION_FIELD_SIZE); - } + public String getAction() { return action; } /** * @param action @@ -380,9 +378,7 @@ public class AuthzAuditEvent extends AuditEventBase { /** * @return the requestData */ - public String getRequestData() { - return trim(requestData, MAX_REQUEST_DATA_FIELD_SIZE); - } + public String getRequestData() { return requestData; } /** * @param requestData http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/33f84070/agents-audit/src/main/java/org/apache/ranger/audit/provider/DbAuditProvider.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/DbAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/DbAuditProvider.java index 98da1c2..f23f17d 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/DbAuditProvider.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/DbAuditProvider.java @@ -32,6 +32,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.audit.dao.DaoManager; import org.apache.ranger.audit.destination.AuditDestination; +import org.apache.ranger.audit.entity.AuthzAuditEventDbObj; import org.apache.ranger.audit.model.AuditEventBase; import org.apache.ranger.audit.model.AuthzAuditEvent; import org.apache.ranger.authorization.hadoop.utils.RangerCredentialProvider; @@ -91,6 +92,9 @@ public class DbAuditProvider extends AuditDestination { if(jdbcPassword != null && !jdbcPassword.isEmpty()) { mDbProperties.put(AUDIT_JPA_JDBC_PASSWORD, jdbcPassword); } + + // initialize the database related classes + AuthzAuditEventDbObj.init(props); } @Override
