Repository: incubator-atlas Updated Branches: refs/heads/master d519ae8cf -> 64de52336
ATLAS-106 Store createTimestamp and modified timestamp separately for an entity (dkantor via shwethags) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/64de5233 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/64de5233 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/64de5233 Branch: refs/heads/master Commit: 64de5233600b2f4a837415d148a3c3aa7234c7a4 Parents: d519ae8 Author: Shwetha GS <[email protected]> Authored: Wed Jan 20 12:33:05 2016 +0530 Committer: Shwetha GS <[email protected]> Committed: Wed Jan 20 12:40:51 2016 +0530 ---------------------------------------------------------------------- release-log.txt | 1 + .../org/apache/atlas/repository/Constants.java | 2 + .../graph/GraphBackedMetadataRepository.java | 4 +- .../graph/TypedInstanceToGraphMapper.java | 4 +- .../GraphBackedDiscoveryServiceTest.java | 47 +++++++++++++++++++- ...kedMetadataRepositoryDeleteEntitiesTest.java | 26 +++++------ .../GraphBackedMetadataRepositoryTest.java | 41 +++++++++++++++++ 7 files changed, 106 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/64de5233/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index e4346d0..52c0584 100644 --- a/release-log.txt +++ b/release-log.txt @@ -7,6 +7,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ALL CHANGES: +ATLAS-106 Store createTimestamp and modified timestamp separately for an entity (dkantor via shwethags) ATLAS-433 Fix checkstyle issues for common and notification module (shwethags) ATLAS-183 Add a Hook in Storm to post the topology metadata (svenkat,yhemanth via shwethags) ATLAS-370 Implement deleteEntities at repository level (dkantor via shwethags) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/64de5233/repository/src/main/java/org/apache/atlas/repository/Constants.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/Constants.java b/repository/src/main/java/org/apache/atlas/repository/Constants.java index d5d3b3c..1b4cb36 100755 --- a/repository/src/main/java/org/apache/atlas/repository/Constants.java +++ b/repository/src/main/java/org/apache/atlas/repository/Constants.java @@ -57,6 +57,8 @@ public final class Constants { public static final String VERSION_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "version"; public static final String TIMESTAMP_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "timestamp"; + public static final String MODIFICATION_TIMESTAMP_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "modificationTimestamp"; + /** * search backing index name. http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/64de5233/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java index d2f6103..e002545 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java @@ -224,7 +224,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository { // update the traits in entity once adding trait instance is successful GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName); - + GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.valueOf(System.currentTimeMillis())); + } catch (RepositoryException e) { throw e; } catch (Exception e) { @@ -285,6 +286,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { for (String traitName : traitNames) { GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName); } + GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.valueOf(System.currentTimeMillis())); } @Override http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/64de5233/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java index dda2b5c..8668b46 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java @@ -21,7 +21,6 @@ import com.thinkaurelius.titan.core.SchemaViolationException; import com.tinkerpop.blueprints.Direction; import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Vertex; - import org.apache.atlas.AtlasException; import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.RepositoryException; @@ -166,8 +165,9 @@ public final class TypedInstanceToGraphMapper { if (Operation.CREATE.equals(operation)) { //TODO - Handle Trait updates addTraits(typedInstance, instanceVertex, classType); + } else if (Operation.UPDATE_FULL.equals(operation) || Operation.UPDATE_PARTIAL.equals(operation)) { + GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.valueOf(System.currentTimeMillis())); } - return getId(typedInstance)._getId(); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/64de5233/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java index 74813a1..bb4e61f 100755 --- a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java @@ -24,10 +24,12 @@ import org.apache.atlas.BaseHiveRepositoryTest; import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.TestUtils; import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; +import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.MetadataRepository; import org.apache.atlas.repository.graph.GraphProvider; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.Referenceable; +import org.apache.atlas.typesystem.persistence.Id; import org.apache.atlas.typesystem.types.ClassType; import org.apache.atlas.typesystem.types.DataTypes; import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; @@ -43,6 +45,10 @@ import org.testng.annotations.Guice; import org.testng.annotations.Test; import javax.inject.Inject; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef; import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef; @@ -71,6 +77,13 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest { ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED); repositoryService.createEntities(hrDept2); + + ITypedReferenceableInstance jane = repositoryService.getEntityDefinition("Person", "name", "Jane"); + Id janeGuid = jane.getId(); + ClassType personType = typeSystem.getDataType(ClassType.class, "Person"); + ITypedReferenceableInstance instance = personType.createInstance(janeGuid); + instance.set("orgLevel", "L1"); + repositoryService.updateEntities(instance); } @AfterClass @@ -115,15 +128,45 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest { public void testRawSearch1() throws Exception { // Query for all Vertices in Graph Object r = discoveryService.searchByGremlin("g.V.toList()"); + Assert.assertTrue(r instanceof List); + List<Map<String, Object>> resultList = (List<Map<String, Object>>) r; + Assert.assertTrue(resultList.size() > 0); System.out.println("search result = " + r); // Query for all Vertices of a Type - r = discoveryService.searchByGremlin("g.V.filter{it.typeName == 'Department'}.toList()"); + r = discoveryService.searchByGremlin("g.V.filter{it." + Constants.ENTITY_TYPE_PROPERTY_KEY + " == 'Department'}.toList()"); + Assert.assertTrue(r instanceof List); + resultList = (List<Map<String, Object>>) r; + Assert.assertTrue(resultList.size() > 0); System.out.println("search result = " + r); // Property Query: list all Person names - r = discoveryService.searchByGremlin("g.V.filter{it.typeName == 'Person'}.'Person.name'.toList()"); + r = discoveryService.searchByGremlin("g.V.filter{it." + Constants.ENTITY_TYPE_PROPERTY_KEY + " == 'Person'}.'Person.name'.toList()"); + Assert.assertTrue(r instanceof List); + resultList = (List<Map<String, Object>>) r; + Assert.assertTrue(resultList.size() > 0); System.out.println("search result = " + r); + List<Object> names = new ArrayList<Object>(resultList.size()); + for (Map<String, Object> vertexProps : resultList) { + names.addAll(vertexProps.values()); + } + for (String name : Arrays.asList("John", "Max")) { + Assert.assertTrue(names.contains(name)); + } + + // Query for all Vertices modified after 01/01/2015 00:00:00 GMT + r = discoveryService.searchByGremlin("g.V.filter{it." + Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY + " > 1420070400000}.toList()"); + Assert.assertTrue(r instanceof List); + resultList = (List<Map<String, Object>>) r; + Assert.assertTrue(resultList.size() > 0); + for (Map<String, Object> vertexProps : resultList) { + Object object = vertexProps.get(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY); + Assert.assertNotNull(object); + Long timestampAsLong = Long.valueOf((String)object); + Assert.assertTrue(timestampAsLong > 1420070400000L); + object = vertexProps.get(Constants.TIMESTAMP_PROPERTY_KEY); + Assert.assertNotNull(object); + } } @DataProvider(name = "dslQueriesProvider") http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/64de5233/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java index e63f6d3..e199913 100644 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java @@ -18,11 +18,9 @@ package org.apache.atlas.repository.graph; -import java.util.ArrayList; -import java.util.List; - -import javax.inject.Inject; - +import com.thinkaurelius.titan.core.TitanGraph; +import com.thinkaurelius.titan.core.util.TitanCleanup; +import com.tinkerpop.blueprints.Vertex; import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.TestUtils; import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; @@ -40,9 +38,9 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Guice; import org.testng.annotations.Test; -import com.thinkaurelius.titan.core.TitanGraph; -import com.thinkaurelius.titan.core.util.TitanCleanup; -import com.tinkerpop.blueprints.Vertex; +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; /** * Test for GraphBackedMetadataRepository.deleteEntities @@ -102,7 +100,8 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest { Assert.assertTrue(refValue instanceof List); List<Object> employees = (List<Object>)refValue; Assert.assertEquals(employees.size(), 4); - List<String> employeeGuids = new ArrayList<String>(4); + + List<String> employeeGuids = new ArrayList(4); for (Object listValue : employees) { Assert.assertTrue(listValue instanceof ITypedReferenceableInstance); ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue; @@ -123,6 +122,7 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest { for (String employeeGuid : employeeGuids) { verifyEntityDoesNotExist(employeeGuid); } + // Verify all Person.address struct vertices were removed. vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "Address"); Assert.assertEquals(vertexCount, 0); @@ -159,11 +159,11 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest { List<String> guids = repositoryService.createEntities(hrDept2); Assert.assertNotNull(guids); Assert.assertEquals(guids.size(), 5); + List<String> entityList = repositoryService.getEntityList("Department"); Assert.assertNotNull(entityList); Assert.assertEquals(entityList.size(), 1); - String hrDeptGuid = entityList.get(0); - return hrDeptGuid; + return entityList.get(0); } private int countVertices(String propertyName, Object value) { @@ -176,12 +176,10 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest { } private void verifyEntityDoesNotExist(String hrDeptGuid) throws RepositoryException { - try { repositoryService.getEntityDefinition(hrDeptGuid); Assert.fail("EntityNotFoundException was expected but none thrown"); - } - catch(EntityNotFoundException e) { + } catch(EntityNotFoundException e) { // good } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/64de5233/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java index a23556b..4083dc5 100755 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java @@ -24,6 +24,7 @@ import com.thinkaurelius.titan.core.util.TitanCleanup; import com.tinkerpop.blueprints.Compare; import com.tinkerpop.blueprints.GraphQuery; import com.tinkerpop.blueprints.Vertex; + import org.apache.atlas.GraphTransaction; import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.TestUtils; @@ -52,9 +53,11 @@ import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Guice; import org.testng.annotations.Test; + import scala.actors.threadpool.Arrays; import javax.inject.Inject; + import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -237,6 +240,9 @@ public class GraphBackedMetadataRepositoryTest { @Test(dependsOnMethods = "testGetTraitNames") public void testAddTrait() throws Exception { final String aGUID = getGUID(); + Vertex vertex = GraphHelper.getInstance().getVertexForGUID(aGUID); + Long modificationTimestampPreUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY); + Assert.assertNull(modificationTimestampPreUpdate); List<String> traitNames = repositoryService.getTraitNames(aGUID); System.out.println("traitNames = " + traitNames); @@ -254,6 +260,11 @@ public class GraphBackedMetadataRepositoryTest { Assert.assertEquals(traitNames.size(), 2); Assert.assertTrue(traitNames.contains(TestUtils.PII)); Assert.assertTrue(traitNames.contains(TestUtils.CLASSIFICATION)); + + // Verify modification timestamp was updated. + GraphHelper.getInstance().getVertexForGUID(aGUID); + Long modificationTimestampPostUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY); + Assert.assertNotNull(modificationTimestampPostUpdate); } @Test(dependsOnMethods = "testAddTrait") @@ -301,6 +312,9 @@ public class GraphBackedMetadataRepositoryTest { @Test(dependsOnMethods = "testAddTrait") public void testDeleteTrait() throws Exception { final String aGUID = getGUID(); + Vertex vertex = GraphHelper.getInstance().getVertexForGUID(aGUID); + Long modificationTimestampPreUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY); + Assert.assertNotNull(modificationTimestampPreUpdate); List<String> traitNames = repositoryService.getTraitNames(aGUID); Assert.assertEquals(traitNames.size(), 3); @@ -315,6 +329,12 @@ public class GraphBackedMetadataRepositoryTest { Assert.assertEquals(traitNames.size(), 2); Assert.assertTrue(traitNames.contains(TestUtils.CLASSIFICATION)); Assert.assertFalse(traitNames.contains(TestUtils.PII)); + + // Verify modification timestamp was updated. + GraphHelper.getInstance().getVertexForGUID(aGUID); + Long modificationTimestampPostUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY); + Assert.assertNotNull(modificationTimestampPostUpdate); + Assert.assertTrue(modificationTimestampPostUpdate > modificationTimestampPreUpdate); } @Test(expectedExceptions = RepositoryException.class) @@ -479,8 +499,16 @@ public class GraphBackedMetadataRepositoryTest { public void testUpdateEntity_MultiplicityOneNonCompositeReference() throws Exception { ITypedReferenceableInstance john = repositoryService.getEntityDefinition("Person", "name", "John"); Id johnGuid = john.getId(); + ITypedReferenceableInstance max = repositoryService.getEntityDefinition("Person", "name", "Max"); String maxGuid = max.getId()._getId(); + Vertex vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid); + Long creationTimestamp = vertex.getProperty(Constants.TIMESTAMP_PROPERTY_KEY); + Assert.assertNotNull(creationTimestamp); + + Long modificationTimestampPreUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY); + Assert.assertNull(modificationTimestampPreUpdate); + ITypedReferenceableInstance jane = repositoryService.getEntityDefinition("Person", "name", "Jane"); Id janeGuid = jane.getId(); @@ -497,6 +525,13 @@ public class GraphBackedMetadataRepositoryTest { ITypedReferenceableInstance refTarget = (ITypedReferenceableInstance) object; Assert.assertEquals(refTarget.getId()._getId(), johnGuid._getId()); + + // Verify modification timestamp was updated. + vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid); + Long modificationTimestampPostUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY); + Assert.assertNotNull(modificationTimestampPostUpdate); + Assert.assertTrue(creationTimestamp < modificationTimestampPostUpdate); + // Update max's mentor reference to jane. instance = personType.createInstance(max.getId()); instance.set("mentor", janeGuid); @@ -508,6 +543,12 @@ public class GraphBackedMetadataRepositoryTest { Assert.assertTrue(object instanceof ITypedReferenceableInstance); refTarget = (ITypedReferenceableInstance) object; Assert.assertEquals(refTarget.getId()._getId(), janeGuid._getId()); + + // Verify modification timestamp was updated. + vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid); + Long modificationTimestampPost2ndUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY); + Assert.assertNotNull(modificationTimestampPost2ndUpdate); + Assert.assertTrue(modificationTimestampPostUpdate < modificationTimestampPost2ndUpdate); } private ITypedReferenceableInstance createHiveTableInstance(Referenceable databaseInstance) throws Exception {
