This is an automated email from the ASF dual-hosted git repository. madhan pushed a commit to branch RANGER-3923 in repository https://gitbox.apache.org/repos/asf/ranger.git
commit 9e05af16b7a06cb7970ce42b215fe3ad2cd0ba05 Author: Madhan Neethiraj <mad...@apache.org> AuthorDate: Mon Nov 20 13:34:22 2023 -0800 RANGER-4282: updated audit logs to capture datasets and projects --- .../destination/ElasticSearchAuditDestination.java | 2 ++ .../audit/destination/SolrAuditDestination.java | 2 ++ .../apache/ranger/audit/model/AuthzAuditEvent.java | 37 +++++++++++++++++----- .../audit/provider/solr/SolrAuditProvider.java | 2 ++ .../plugin/audit/RangerDefaultAuditHandler.java | 17 ++++++++++ .../config/solr-ranger_audits/managed-schema | 4 ++- .../conf/ranger_es_schema.json | 8 ++++- .../solr_for_audit_setup/conf/managed-schema | 4 ++- .../cloudwatch/CloudWatchAccessAuditsService.java | 10 ++++++ .../ElasticSearchAccessAuditsService.java | 8 +++++ .../ranger/solr/SolrAccessAuditsService.java | 8 +++++ .../java/org/apache/ranger/view/VXAccessAudit.java | 27 ++++++++++++++-- 12 files changed, 116 insertions(+), 13 deletions(-) diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/ElasticSearchAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/ElasticSearchAuditDestination.java index 1b17a934b..c06002d35 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/ElasticSearchAuditDestination.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/ElasticSearchAuditDestination.java @@ -334,6 +334,8 @@ public class ElasticSearchAuditDestination extends AuditDestination { doc.put("event_count", auditEvent.getEventCount()); doc.put("event_dur_ms", auditEvent.getEventDurationMS()); doc.put("tags", auditEvent.getTags()); + doc.put("datasets", auditEvent.getDatasets()); + doc.put("projects", auditEvent.getProjects()); doc.put("cluster", auditEvent.getClusterName()); doc.put("zoneName", auditEvent.getZoneName()); doc.put("agentHost", auditEvent.getAgentHostname()); diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java index f2e96bf9b..6fe92498a 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java @@ -294,6 +294,8 @@ public class SolrAuditDestination extends AuditDestination { doc.setField("event_count", auditEvent.getEventCount()); doc.setField("event_dur_ms", auditEvent.getEventDurationMS()); doc.setField("tags", auditEvent.getTags()); + doc.addField("datasets", auditEvent.getDatasets()); + doc.addField("projects", auditEvent.getProjects()); doc.setField("cluster", auditEvent.getClusterName()); doc.setField("zoneName", auditEvent.getZoneName()); doc.setField("agentHost", auditEvent.getAgentHostname()); diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java b/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java index e20d1a786..2e32fb579 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java @@ -110,6 +110,12 @@ public class AuthzAuditEvent extends AuditEventBase { @SerializedName("tags") protected Set<String> tags = new HashSet<>(); + @SerializedName("datasets") + protected Set<String> datasets = null; + + @SerializedName("projects") + protected Set<String> projects = null; + @SerializedName("additional_info") protected String additionalInfo; @@ -472,18 +478,34 @@ public class AuthzAuditEvent extends AuditEventBase { return eventDurationMS; } - public Set<String> getTags() { - return tags; - } - public void setEventDurationMS(long frequencyDurationMS) { this.eventDurationMS = frequencyDurationMS; } + public Set<String> getTags() { + return tags; + } + public void setTags(Set<String> tags) { this.tags = tags; } + public Set<String> getDatasets() { + return datasets; + } + + public void setDatasets(Set<String> datasets) { + this.datasets = datasets; + } + + public Set<String> getProjects() { + return projects; + } + + public void setProjects(Set<String> projects) { + this.projects = projects; + } + public String getClusterName() { return clusterName; } @@ -561,10 +583,9 @@ public class AuthzAuditEvent extends AuditEventBase { .append(FIELD_SEPARATOR).append("event_count=") .append(eventCount).append(FIELD_SEPARATOR) .append("event_dur_ms=").append(eventDurationMS) - .append(FIELD_SEPARATOR) - .append("tags=").append("[") - .append(StringUtils.join(tags, ", ")) - .append("]") + .append(FIELD_SEPARATOR).append("tags=").append("[").append(StringUtils.join(tags, ", ")).append("]") + .append(FIELD_SEPARATOR).append("datasets=").append("[").append(datasets != null ? StringUtils.join(datasets, ", ") : "").append("]") + .append(FIELD_SEPARATOR).append("projects=").append("[").append(projects != null ? StringUtils.join(projects, ", ") : "").append("]") .append(FIELD_SEPARATOR).append("clusterName=").append(clusterName) .append(FIELD_SEPARATOR).append("zoneName=").append(zoneName) .append(FIELD_SEPARATOR).append("policyVersion=").append(policyVersion) diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java index 9cda3f8f3..691cef002 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java @@ -290,6 +290,8 @@ public class SolrAuditProvider extends AuditDestination { doc.addField("action", auditEvent.getAction()); doc.addField("evtTime", auditEvent.getEventTime()); doc.addField("tags", auditEvent.getTags()); + doc.addField("datasets", auditEvent.getDatasets()); + doc.addField("projects", auditEvent.getProjects()); doc.addField("cluster", auditEvent.getClusterName()); doc.addField("zone", auditEvent.getZoneName()); doc.addField("agentHost", auditEvent.getAgentHostname()); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java index c99465d7a..8efea1fba 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java @@ -33,6 +33,7 @@ import org.apache.ranger.authorization.hadoop.constants.RangerHadoopConstants; import org.apache.ranger.authorization.utils.JsonUtils; import org.apache.ranger.plugin.contextenricher.RangerTagForEval; import org.apache.ranger.plugin.policyengine.*; +import org.apache.ranger.plugin.policyengine.gds.GdsAccessResult; import org.apache.ranger.plugin.service.RangerBasePlugin; import org.apache.ranger.plugin.util.JsonUtilsV2; import org.apache.ranger.plugin.util.RangerAccessRequestUtil; @@ -132,10 +133,14 @@ public class RangerDefaultAuditHandler implements RangerAccessResultProcessor { ret.setClientType(request.getClientType()); ret.setSessionId(request.getSessionId()); ret.setAclEnforcer(moduleName); + Set<String> tags = getTags(request); if (tags != null) { ret.setTags(tags); } + + ret.setDatasets(getDatasets(request)); + ret.setProjects(getProjects(request)); ret.setAdditionalInfo(getAdditionalInfo(request)); ret.setClusterName(request.getClusterName()); ret.setZoneName(result.getZoneName()); @@ -263,6 +268,18 @@ public class RangerDefaultAuditHandler implements RangerAccessResultProcessor { return ret; } + public final Set<String> getDatasets(RangerAccessRequest request) { + GdsAccessResult gdsResult = RangerAccessRequestUtil.getGdsResultFromContext(request.getContext()); + + return gdsResult != null ? gdsResult.getDatasets() : null; + } + + public final Set<String> getProjects(RangerAccessRequest request) { + GdsAccessResult gdsResult = RangerAccessRequestUtil.getGdsResultFromContext(request.getContext()); + + return gdsResult != null ? gdsResult.getProjects() : null; + } + public String getAdditionalInfo(RangerAccessRequest request) { if (StringUtils.isBlank(request.getRemoteIPAddress()) && CollectionUtils.isEmpty(request.getForwardedAddresses())) { return null; diff --git a/dev-support/ranger-docker/config/solr-ranger_audits/managed-schema b/dev-support/ranger-docker/config/solr-ranger_audits/managed-schema index c33f6de06..df53a05df 100644 --- a/dev-support/ranger-docker/config/solr-ranger_audits/managed-schema +++ b/dev-support/ranger-docker/config/solr-ranger_audits/managed-schema @@ -92,4 +92,6 @@ <field name="text" type="text_std_token_lower_case" multiValued="true" indexed="true" stored="false"/> <field name="zoneName" type="key_lower_case" multiValued="false"/> <field name="policyVersion" type="tlong" multiValued="false"/> -</schema> \ No newline at end of file + <field name="datasets" type="key_lower_case" multiValued="true"/> + <field name="projects" type="key_lower_case" multiValued="true"/> +</schema> diff --git a/security-admin/contrib/elasticsearch_for_audit_setup/conf/ranger_es_schema.json b/security-admin/contrib/elasticsearch_for_audit_setup/conf/ranger_es_schema.json index 801667bce..d90e18b7d 100644 --- a/security-admin/contrib/elasticsearch_for_audit_setup/conf/ranger_es_schema.json +++ b/security-admin/contrib/elasticsearch_for_audit_setup/conf/ranger_es_schema.json @@ -111,6 +111,12 @@ "tags_str": { "type": "text" }, + "datasets": { + "type": "keyword" + }, + "projects": { + "type": "keyword" + }, "text": { "type": "text" }, @@ -121,4 +127,4 @@ "type": "long" } } -} \ No newline at end of file +} diff --git a/security-admin/contrib/solr_for_audit_setup/conf/managed-schema b/security-admin/contrib/solr_for_audit_setup/conf/managed-schema index c33f6de06..df53a05df 100644 --- a/security-admin/contrib/solr_for_audit_setup/conf/managed-schema +++ b/security-admin/contrib/solr_for_audit_setup/conf/managed-schema @@ -92,4 +92,6 @@ <field name="text" type="text_std_token_lower_case" multiValued="true" indexed="true" stored="false"/> <field name="zoneName" type="key_lower_case" multiValued="false"/> <field name="policyVersion" type="tlong" multiValued="false"/> -</schema> \ No newline at end of file + <field name="datasets" type="key_lower_case" multiValued="true"/> + <field name="projects" type="key_lower_case" multiValued="true"/> +</schema> diff --git a/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchAccessAuditsService.java b/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchAccessAuditsService.java index 0b36f6e90..06a21a3c2 100644 --- a/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchAccessAuditsService.java +++ b/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchAccessAuditsService.java @@ -283,6 +283,16 @@ public class CloudWatchAccessAuditsService extends org.apache.ranger.AccessAudit accessAudit.setTags(value.toString()); } + value = auditEvent.getDatasets(); + if (value != null) { + accessAudit.setDatasets(value.toString()); + } + + value = auditEvent.getProjects(); + if (value != null) { + accessAudit.setProjects(value.toString()); + } + return accessAudit; } diff --git a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java index 4c9b049a0..4195567fa 100644 --- a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java +++ b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java @@ -275,6 +275,14 @@ public class ElasticSearchAccessAuditsService extends org.apache.ranger.AccessAu if (value != null) { accessAudit.setTags(value.toString()); } + value = source.get("datasets"); + if (value != null) { + accessAudit.setDatasets(value.toString()); + } + value = source.get("projects"); + if (value != null) { + accessAudit.setProjects(value.toString()); + } return accessAudit; } diff --git a/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java b/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java index bb279349a..3485f4dba 100644 --- a/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java +++ b/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java @@ -254,6 +254,14 @@ public class SolrAccessAuditsService extends AccessAuditsService { if (value != null) { accessAudit.setTags(value.toString()); } + value = doc.getFieldValue("datasets"); + if (value != null) { + accessAudit.setDatasets(value.toString()); + } + value = doc.getFieldValue("projects"); + if (value != null) { + accessAudit.setProjects(value.toString()); + } return accessAudit; } diff --git a/security-admin/src/main/java/org/apache/ranger/view/VXAccessAudit.java b/security-admin/src/main/java/org/apache/ranger/view/VXAccessAudit.java index cce18fafb..9cf06f93c 100644 --- a/security-admin/src/main/java/org/apache/ranger/view/VXAccessAudit.java +++ b/security-admin/src/main/java/org/apache/ranger/view/VXAccessAudit.java @@ -137,7 +137,11 @@ public class VXAccessAudit extends VXDataObject implements java.io.Serializable protected long eventDuration; protected String tags; - + + protected String datasets; + + protected String projects; + protected String clusterName; // Security Zone @@ -548,7 +552,24 @@ public class VXAccessAudit extends VXDataObject implements java.io.Serializable public void setTags(String tags) { this.tags = tags; } - + + public String getDatasets() { + return datasets; + } + + public void setDatasets(String datasets) { + this.datasets = datasets; + } + + public String getProjects() { + return projects; + } + + public void setProjects(String projects) { + this.projects = projects; + } + + /** * @return the clusterName */ @@ -645,6 +666,8 @@ public class VXAccessAudit extends VXDataObject implements java.io.Serializable str += "eventCount={" + eventCount + "}"; str += "eventDuration={" + eventDuration + "}"; str += "tags={" + tags + "}"; + str += "datasets={" + datasets + "}"; + str += "projects={" + projects + "}"; str += "clusterName={" + clusterName + "}"; str += "zoneName={" + zoneName + "}"; str += "agentHost={" + agentHost + "}";