ATLAS-1087 Provide an option to turn off persisting entity definition in audits (sumasai, shwethags)
(cherry picked from commit 1abd5a248ef424642aef2c40ce0bb7abbbc900ce) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/41ea3f54 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/41ea3f54 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/41ea3f54 Branch: refs/heads/0.7-incubating Commit: 41ea3f54789a7164da28133db7751138784bbfa0 Parents: 269bc9a Author: Suma Shivaprasad <[email protected]> Authored: Fri Aug 5 11:33:25 2016 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Thu Dec 22 14:50:45 2016 -0800 ---------------------------------------------------------------------- release-log.txt | 1 + .../audit/HBaseBasedAuditRepository.java | 28 +++++++++++++++++--- .../audit/AuditRepositoryTestBase.java | 3 ++- .../audit/HBaseBasedAuditRepositoryTest.java | 14 +++++++--- 4 files changed, 39 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/41ea3f54/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index e39c69c..07b6ca0 100644 --- a/release-log.txt +++ b/release-log.txt @@ -28,6 +28,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ALL CHANGES: +ATLAS-1087 Provide an option to turn off persisting entity definition in audits (sumasai, shwethags) ATLAS-1097 Fix a potential NPE issue flagged by Coverity scan (mneethiraj via shwethags) ATLAS-1090 UI: Multi-Select Tagging. (Kalyanikashikar via kevalbhatt) ATLAS-1092 Add Table.CreateTime to process qualified Name for all hive_process (sumasai via shwethags) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/41ea3f54/repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java index 22d71df..0ecbb3b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java @@ -74,12 +74,23 @@ public class HBaseBasedAuditRepository implements Service, EntityAuditRepository private static final String FIELD_SEPARATOR = ":"; + public static final String CONFIG_PERSIST_ENTITY_DEFINITION = CONFIG_PREFIX + ".persistEntityDefinition"; + public static final byte[] COLUMN_FAMILY = Bytes.toBytes("dt"); public static final byte[] COLUMN_ACTION = Bytes.toBytes("action"); public static final byte[] COLUMN_DETAIL = Bytes.toBytes("detail"); public static final byte[] COLUMN_USER = Bytes.toBytes("user"); public static final byte[] COLUMN_DEFINITION = Bytes.toBytes("def"); + private static boolean persistEntityDefinition; + + static { + try { + persistEntityDefinition = ApplicationProperties.get().getBoolean(CONFIG_PERSIST_ENTITY_DEFINITION, false); + } catch (AtlasException e) { + throw new RuntimeException(e); + } + } private TableName tableName; private Connection connection; @@ -111,7 +122,9 @@ public class HBaseBasedAuditRepository implements Service, EntityAuditRepository addColumn(put, COLUMN_ACTION, event.getAction()); addColumn(put, COLUMN_USER, event.getUser()); addColumn(put, COLUMN_DETAIL, event.getDetails()); - addColumn(put, COLUMN_DEFINITION, event.getEntityDefinitionString()); + if (persistEntityDefinition) { + addColumn(put, COLUMN_DEFINITION, event.getEntityDefinitionString()); + } puts.add(put); } table.put(puts); @@ -185,7 +198,12 @@ public class HBaseBasedAuditRepository implements Service, EntityAuditRepository event.setUser(getResultString(result, COLUMN_USER)); event.setAction(EntityAuditEvent.EntityAuditAction.valueOf(getResultString(result, COLUMN_ACTION))); event.setDetails(getResultString(result, COLUMN_DETAIL)); - event.setEntityDefinition(getResultString(result, COLUMN_DEFINITION)); + if (persistEntityDefinition) { + String colDef = getResultString(result, COLUMN_DEFINITION); + if (colDef != null) { + event.setEntityDefinition(colDef); + } + } events.add(event); } LOG.info("Got events for entity id {}, starting timestamp {}, #records {}", entityId, startKey, events.size()); @@ -199,7 +217,11 @@ public class HBaseBasedAuditRepository implements Service, EntityAuditRepository } private String getResultString(Result result, byte[] columnName) { - return Bytes.toString(result.getValue(COLUMN_FAMILY, columnName)); + byte[] rawValue = result.getValue(COLUMN_FAMILY, columnName); + if ( rawValue != null) { + return Bytes.toString(rawValue); + } + return null; } private EntityAuditEvent fromKey(byte[] keyBytes) { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/41ea3f54/repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java b/repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java index f699404..7ae5e20 100644 --- a/repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java +++ b/repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java @@ -18,6 +18,7 @@ package org.apache.atlas.repository.audit; +import junit.framework.Assert; import org.apache.atlas.EntityAuditEvent; import org.apache.atlas.typesystem.Referenceable; import org.apache.commons.lang.RandomStringUtils; @@ -88,7 +89,7 @@ public class AuditRepositoryTestBase { assertEquals(events.size(), 0); } - private void assertEventEquals(EntityAuditEvent actual, EntityAuditEvent expected) { + protected void assertEventEquals(EntityAuditEvent actual, EntityAuditEvent expected) { if (expected != null) { assertNotNull(actual); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/41ea3f54/repository/src/test/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepositoryTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepositoryTest.java b/repository/src/test/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepositoryTest.java index 677eb39..79550ba 100644 --- a/repository/src/test/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepositoryTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepositoryTest.java @@ -19,6 +19,7 @@ package org.apache.atlas.repository.audit; import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.EntityAuditEvent; import org.apache.commons.configuration.Configuration; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; @@ -27,6 +28,7 @@ import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; public class HBaseBasedAuditRepositoryTest extends AuditRepositoryTestBase { @@ -36,7 +38,7 @@ public class HBaseBasedAuditRepositoryTest extends AuditRepositoryTestBase { public void setup() throws Exception { eventRepository = new HBaseBasedAuditRepository(); HBaseTestUtils.startCluster(); - ((HBaseBasedAuditRepository)eventRepository).start(); + ((HBaseBasedAuditRepository) eventRepository).start(); Configuration properties = ApplicationProperties.get(); String tableNameStr = properties.getString(HBaseBasedAuditRepository.CONFIG_TABLE_NAME, @@ -46,7 +48,7 @@ public class HBaseBasedAuditRepositoryTest extends AuditRepositoryTestBase { @AfterClass public void teardown() throws Exception { - ((HBaseBasedAuditRepository)eventRepository).stop(); + ((HBaseBasedAuditRepository) eventRepository).stop(); HBaseTestUtils.stopCluster(); } @@ -56,4 +58,10 @@ public class HBaseBasedAuditRepositoryTest extends AuditRepositoryTestBase { Admin admin = connection.getAdmin(); assertTrue(admin.tableExists(tableName)); } -} + + @Override + protected void assertEventEquals(EntityAuditEvent actual, EntityAuditEvent expected) { + super.assertEventEquals(actual, expected); + assertNull(actual.getEntityDefinition()); + } +} \ No newline at end of file
