http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/40ee9492/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java index b94ff5a..de29e86 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java @@ -19,14 +19,14 @@ package org.apache.atlas.repository.graph; import com.google.common.base.Preconditions; +import com.google.inject.Inject; +import com.google.inject.Singleton; import com.thinkaurelius.titan.core.TitanGraph; -import com.tinkerpop.blueprints.Direction; -import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.GraphQuery; import com.tinkerpop.blueprints.Vertex; - import org.apache.atlas.AtlasException; import org.apache.atlas.GraphTransaction; +import org.apache.atlas.RequestContext; import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.MetadataRepository; import org.apache.atlas.repository.RepositoryException; @@ -37,15 +37,13 @@ import org.apache.atlas.typesystem.exception.EntityNotFoundException; import org.apache.atlas.typesystem.exception.TraitNotFoundException; import org.apache.atlas.typesystem.types.AttributeInfo; import org.apache.atlas.typesystem.types.ClassType; +import org.apache.atlas.typesystem.types.DataTypes; import org.apache.atlas.typesystem.types.IDataType; import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.TypeUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; -import javax.inject.Singleton; - import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; @@ -60,18 +58,21 @@ public class GraphBackedMetadataRepository implements MetadataRepository { private static final Logger LOG = LoggerFactory.getLogger(GraphBackedMetadataRepository.class); - private final GraphToTypedInstanceMapper graphToInstanceMapper; - private static TypeSystem typeSystem = TypeSystem.getInstance(); private static final GraphHelper graphHelper = GraphHelper.getInstance(); private final TitanGraph titanGraph; + private DeleteHandler deleteHandler; + + private GraphToTypedInstanceMapper graphToInstanceMapper; + @Inject - public GraphBackedMetadataRepository(GraphProvider<TitanGraph> graphProvider) { + public GraphBackedMetadataRepository(GraphProvider<TitanGraph> graphProvider, DeleteHandler deleteHandler) { this.titanGraph = graphProvider.get(); - this.graphToInstanceMapper = new GraphToTypedInstanceMapper(titanGraph); + graphToInstanceMapper = new GraphToTypedInstanceMapper(titanGraph); + this.deleteHandler = deleteHandler; } public GraphToTypedInstanceMapper getGraphToInstanceMapper() { @@ -122,10 +123,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository { EntityExistsException { LOG.info("adding entities={}", entities); try { - TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper); - TypeUtils.Pair<List<String>, List<String>> idPair = - instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.CREATE, entities); - return idPair.left; + TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper, deleteHandler); + instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.CREATE, entities); + return RequestContext.get().getCreatedEntityIds(); } catch (EntityExistsException e) { throw e; } catch (AtlasException e) { @@ -215,14 +215,15 @@ public class GraphBackedMetadataRepository implements MetadataRepository { // add the trait instance as a new vertex final String typeName = GraphHelper.getTypeName(instanceVertex); - TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper); + TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper, deleteHandler); instanceToGraphMapper.mapTraitInstanceToVertex(traitInstance, typeSystem.getDataType(ClassType.class, typeName), instanceVertex); // update the traits in entity once adding trait instance is successful GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName); - GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.valueOf(System.currentTimeMillis())); + GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, + RequestContext.get().getRequestTime()); } catch (RepositoryException e) { throw e; @@ -254,23 +255,12 @@ public class GraphBackedMetadataRepository implements MetadataRepository { try { final String entityTypeName = GraphHelper.getTypeName(instanceVertex); String relationshipLabel = GraphHelper.getTraitLabel(entityTypeName, traitNameToBeDeleted); - Iterator<Edge> results = instanceVertex.getEdges(Direction.OUT, relationshipLabel).iterator(); - if (results.hasNext()) { // there should only be one edge for this label - final Edge traitEdge = results.next(); - final Vertex traitVertex = traitEdge.getVertex(Direction.IN); - - // remove the edge to the trait instance from the repository - titanGraph.removeEdge(traitEdge); - - if (traitVertex != null) { // remove the trait instance from the repository - TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper); - instanceToGraphMapper.deleteTraitVertex(traitNameToBeDeleted, traitVertex); - - // update the traits in entity once trait removal is successful - traitNames.remove(traitNameToBeDeleted); - updateTraits(instanceVertex, traitNames); - } - } + + deleteHandler.deleteReference(instanceVertex, relationshipLabel, DataTypes.TypeCategory.TRAIT); + + // update the traits in entity once trait removal is successful + traitNames.remove(traitNameToBeDeleted); + updateTraits(instanceVertex, traitNames); } catch (Exception e) { throw new RepositoryException(e); } @@ -285,7 +275,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository { for (String traitName : traitNames) { GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName); } - GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.valueOf(System.currentTimeMillis())); + GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, + RequestContext.get().getRequestTime()); } @Override @@ -293,9 +284,11 @@ public class GraphBackedMetadataRepository implements MetadataRepository { public TypeUtils.Pair<List<String>, List<String>> updateEntities(ITypedReferenceableInstance... entitiesUpdated) throws RepositoryException { LOG.info("updating entity {}", entitiesUpdated); try { - TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper); - return instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.UPDATE_FULL, + TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper, deleteHandler); + instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.UPDATE_FULL, entitiesUpdated); + RequestContext requestContext = RequestContext.get(); + return TypeUtils.Pair.of(requestContext.getCreatedEntityIds(), requestContext.getUpdatedEntityIds()); } catch (AtlasException e) { throw new RepositoryException(e); } @@ -306,8 +299,10 @@ public class GraphBackedMetadataRepository implements MetadataRepository { public TypeUtils.Pair<List<String>, List<String>> updatePartial(ITypedReferenceableInstance entity) throws RepositoryException { LOG.info("updating entity {}", entity); try { - TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper); - return instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.UPDATE_PARTIAL, entity); + TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper, deleteHandler); + instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.UPDATE_PARTIAL, entity); + RequestContext requestContext = RequestContext.get(); + return TypeUtils.Pair.of(requestContext.getCreatedEntityIds(), requestContext.getUpdatedEntityIds()); } catch (AtlasException e) { throw new RepositoryException(e); } @@ -315,13 +310,12 @@ public class GraphBackedMetadataRepository implements MetadataRepository { @Override @GraphTransaction - public TypeUtils.Pair<List<String>, List<ITypedReferenceableInstance>> deleteEntities(List<String> guids) throws RepositoryException { + public TypeUtils.Pair<List<String>, List<ITypedReferenceableInstance>> deleteEntities(List<String> guids) throws RepositoryException { if (guids == null || guids.size() == 0) { throw new IllegalArgumentException("guids must be non-null and non-empty"); } - TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper); for (String guid : guids) { if (guid == null) { LOG.warn("deleteEntities: Ignoring null guid"); @@ -329,8 +323,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { } try { Vertex instanceVertex = graphHelper.getVertexForGUID(guid); - String typeName = GraphHelper.getTypeName(instanceVertex); - instanceToGraphMapper.deleteEntity(typeName, instanceVertex); + deleteHandler.deleteEntity(instanceVertex); } catch (EntityNotFoundException e) { // Entity does not exist - treat as non-error, since the caller // wanted to delete the entity and it's already gone. @@ -340,7 +333,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { throw new RepositoryException(e); } } - return new TypeUtils.Pair<>( - instanceToGraphMapper.getDeletedEntityGuids(), instanceToGraphMapper.getDeletedEntities()); + RequestContext requestContext = RequestContext.get(); + return new TypeUtils.Pair<>(requestContext.getDeletedEntityIds(), requestContext.getDeletedEntities()); } }
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/40ee9492/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java index e7e8fb9..d83c08c 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java @@ -100,7 +100,11 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang management.buildIndex(Constants.EDGE_INDEX, Edge.class).buildMixedIndex(Constants.BACKING_INDEX); // create a composite index for guid as its unique - createCompositeAndMixedIndex(management, Constants.GUID_PROPERTY_KEY, String.class, true, Cardinality.SINGLE, true); + createCompositeAndMixedIndex(management, Constants.GUID_PROPERTY_KEY, String.class, true, + Cardinality.SINGLE, true); + + // create a composite index for entity state + createCompositeAndMixedIndex(management, Constants.STATE_PROPERTY_KEY, String.class, false, Cardinality.SINGLE, true); // create a composite and mixed index for type since it can be combined with other keys createCompositeAndMixedIndex(management, Constants.ENTITY_TYPE_PROPERTY_KEY, String.class, false, Cardinality.SINGLE, @@ -223,13 +227,13 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang switch (field.dataType().getTypeCategory()) { case PRIMITIVE: Cardinality cardinality = getCardinality(field.multiplicity); - createCompositeAndMixedIndex(management, propertyName, getPrimitiveClass(field.dataType()), field.isUnique, + createCompositeAndMixedIndex(management, propertyName, getPrimitiveClass(field.dataType()), false, cardinality, false); break; case ENUM: cardinality = getCardinality(field.multiplicity); - createCompositeAndMixedIndex(management, propertyName, String.class, field.isUnique, cardinality, false); + createCompositeAndMixedIndex(management, propertyName, String.class, false, cardinality, false); break; case ARRAY: http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/40ee9492/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index 895f9df..c542ec7 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -23,11 +23,12 @@ import com.thinkaurelius.titan.core.TitanProperty; import com.thinkaurelius.titan.core.TitanVertex; import com.tinkerpop.blueprints.Direction; import com.tinkerpop.blueprints.Edge; +import com.tinkerpop.blueprints.Element; import com.tinkerpop.blueprints.Graph; import com.tinkerpop.blueprints.GraphQuery; import com.tinkerpop.blueprints.Vertex; - import org.apache.atlas.AtlasException; +import org.apache.atlas.RequestContext; import org.apache.atlas.repository.Constants; import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.atlas.typesystem.ITypedInstance; @@ -40,7 +41,6 @@ import org.apache.atlas.typesystem.types.DataTypes; import org.apache.atlas.typesystem.types.HierarchicalType; import org.apache.atlas.typesystem.types.IDataType; import org.apache.atlas.typesystem.types.TypeSystem; -import org.apache.atlas.typesystem.types.TypeUtils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -77,7 +77,7 @@ public final class GraphHelper { final String guid = UUID.randomUUID().toString(); final Vertex vertexWithIdentity = createVertexWithoutIdentity(typedInstance.getTypeName(), - new Id(guid, 0 , typedInstance.getTypeName()), superTypeNames); + new Id(guid, 0, typedInstance.getTypeName()), superTypeNames); // add identity setProperty(vertexWithIdentity, Constants.GUID_PROPERTY_KEY, guid); @@ -85,9 +85,6 @@ public final class GraphHelper { // add version information setProperty(vertexWithIdentity, Constants.VERSION_PROPERTY_KEY, typedInstance.getId().version); - // add state information - setProperty(vertexWithIdentity, Constants.STATE_PROPERTY_KEY, Id.EntityState.ACTIVE.name()); - return vertexWithIdentity; } @@ -99,41 +96,120 @@ public final class GraphHelper { // add type information setProperty(vertexWithoutIdentity, Constants.ENTITY_TYPE_PROPERTY_KEY, typeName); + // add super types for (String superTypeName : superTypeNames) { addProperty(vertexWithoutIdentity, Constants.SUPER_TYPES_PROPERTY_KEY, superTypeName); } + // add state information + setProperty(vertexWithoutIdentity, Constants.STATE_PROPERTY_KEY, Id.EntityState.ACTIVE.name()); + // add timestamp information - setProperty(vertexWithoutIdentity, Constants.TIMESTAMP_PROPERTY_KEY, System.currentTimeMillis()); + setProperty(vertexWithoutIdentity, Constants.TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime()); return vertexWithoutIdentity; } public Edge addEdge(Vertex fromVertex, Vertex toVertex, String edgeLabel) { - LOG.debug("Adding edge for {} -> label {} -> {}", fromVertex, edgeLabel, toVertex); + LOG.debug("Adding edge for {} -> label {} -> {}", string(fromVertex), edgeLabel, string(toVertex)); Edge edge = titanGraph.addEdge(null, fromVertex, toVertex, edgeLabel); - LOG.debug("Added edge for {} -> label {}, id {} -> {}", fromVertex, edgeLabel, edge.getId(), toVertex); + + setProperty(edge, Constants.STATE_PROPERTY_KEY, Id.EntityState.ACTIVE.name()); + setProperty(edge, Constants.TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime()); + setProperty(edge, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime()); + + LOG.debug("Added {}", string(edge)); return edge; } - public Vertex findVertex(String propertyKey, Object value) { - LOG.debug("Finding vertex for {}={}", propertyKey, value); + public Edge getOrCreateEdge(Vertex outVertex, Vertex inVertex, String edgeLabel) { + Iterable<Edge> edges = inVertex.getEdges(Direction.IN, edgeLabel); + for (Edge edge : edges) { + if (edge.getVertex(Direction.OUT).getId().toString().equals(outVertex.getId().toString())) { + return edge; + } + } + return addEdge(outVertex, inVertex, edgeLabel); + } + + /** + * Args of the format prop1, key1, prop2, key2... + * Searches for a vertex with prop1=key1 && prop2=key2 + * @param args + * @return vertex with the given property keys + * @throws EntityNotFoundException + */ + private Vertex findVertex(Object... args) throws EntityNotFoundException { + StringBuilder condition = new StringBuilder(); + GraphQuery query = titanGraph.query(); + for (int i = 0 ; i < args.length; i+=2) { + query = query.has((String) args[i], args[i+1]); + condition.append(args[i]).append(" = ").append(args[i+1]).append(", "); + } + String conditionStr = condition.toString(); + LOG.debug("Finding vertex with {}", conditionStr); - GraphQuery query = titanGraph.query().has(propertyKey, value); Iterator<Vertex> results = query.vertices().iterator(); // returning one since entityType, qualifiedName should be unique - return results.hasNext() ? results.next() : null; + Vertex vertex = results.hasNext() ? results.next() : null; + + if (vertex == null) { + LOG.debug("Could not find a vertex with {}", condition.toString()); + throw new EntityNotFoundException("Could not find an entity in the repository with " + conditionStr); + } else { + LOG.debug("Found a vertex {} with {}", string(vertex), conditionStr); + } + + return vertex; } - public static Iterable<Edge> getOutGoingEdgesByLabel(Vertex instanceVertex, String edgeLabel) { + public static Iterator<Edge> getOutGoingEdgesByLabel(Vertex instanceVertex, String edgeLabel) { + LOG.debug("Finding edges for {} with label {}", string(instanceVertex), edgeLabel); if(instanceVertex != null && edgeLabel != null) { - return instanceVertex.getEdges(Direction.OUT, edgeLabel); + return instanceVertex.getEdges(Direction.OUT, edgeLabel).iterator(); } return null; } - public Edge getOutGoingEdgeById(String edgeId) { + /** + * Returns the active edge for the given edge label. + * If the vertex is deleted and there is no active edge, it returns the latest deleted edge + * @param vertex + * @param edgeLabel + * @return + */ + public static Edge getEdgeForLabel(Vertex vertex, String edgeLabel) { + String vertexState = vertex.getProperty(Constants.STATE_PROPERTY_KEY); + + Iterator<Edge> iterator = GraphHelper.getOutGoingEdgesByLabel(vertex, edgeLabel); + Edge latestDeletedEdge = null; + long latestDeletedEdgeTime = Long.MIN_VALUE; + while (iterator != null && iterator.hasNext()) { + Edge edge = iterator.next(); + String edgeState = edge.getProperty(Constants.STATE_PROPERTY_KEY); + if (edgeState == null || Id.EntityState.ACTIVE.name().equals(edgeState)) { + LOG.debug("Found {}", string(edge)); + return edge; + } else { + Long modificationTime = edge.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY); + if (modificationTime != null && modificationTime >= latestDeletedEdgeTime) { + latestDeletedEdgeTime = modificationTime; + latestDeletedEdge = edge; + } + } + } + + //If the vertex is deleted, return latest deleted edge + if (Id.EntityState.DELETED.equals(vertexState)) { + LOG.debug("Found {}", string(latestDeletedEdge)); + return latestDeletedEdge; + } + + return null; + } + + public Edge getEdgeById(String edgeId) { if(edgeId != null) { return titanGraph.getEdge(edgeId); } @@ -154,60 +230,47 @@ public final class GraphHelper { + edge.getVertex(Direction.IN) + "]"; } - public static void setProperty(Vertex vertex, String propertyName, Object value) { - LOG.debug("Setting property {} = \"{}\" to vertex {}", propertyName, value, vertex); - Object existValue = vertex.getProperty(propertyName); + public static <T extends Element> void setProperty(T element, String propertyName, Object value) { + String elementStr = string(element); + LOG.debug("Setting property {} = \"{}\" to {}", propertyName, value, elementStr); + Object existValue = element.getProperty(propertyName); if(value == null || (value instanceof Collection && ((Collection) value).isEmpty())) { if(existValue != null) { - LOG.info("Removing property - {} value from vertex {}", propertyName, vertex); - vertex.removeProperty(propertyName); + LOG.info("Removing property - {} value from {}", propertyName, elementStr); + element.removeProperty(propertyName); } } else { if (!value.equals(existValue)) { - vertex.setProperty(propertyName, value); - LOG.debug("Set property {} = \"{}\" to vertex {}", propertyName, value, vertex); + element.setProperty(propertyName, value); + LOG.debug("Set property {} = \"{}\" to {}", propertyName, value, elementStr); } } } + private static <T extends Element> String string(T element) { + if (element instanceof Vertex) { + return string((Vertex) element); + } else if (element instanceof Edge) { + return string((Edge)element); + } + return element.toString(); + } + public static void addProperty(Vertex vertex, String propertyName, Object value) { - LOG.debug("Setting property {} = \"{}\" to vertex {}", propertyName, value, vertex); + LOG.debug("Adding property {} = \"{}\" to vertex {}", propertyName, value, string(vertex)); ((TitanVertex)vertex).addProperty(propertyName, value); } - public Edge removeRelation(String edgeId, boolean cascade) { - LOG.debug("Removing edge with id {}", edgeId); - final Edge edge = titanGraph.getEdge(edgeId); - titanGraph.removeEdge(edge); - LOG.info("Removed edge {}", edge); - if (cascade) { - Vertex referredVertex = edge.getVertex(Direction.IN); - removeVertex(referredVertex); - } - return edge; - } - /** * Remove the specified edge from the graph. * * @param edge */ public void removeEdge(Edge edge) { - LOG.debug("Removing edge {}", edge); + String edgeString = string(edge); + LOG.debug("Removing {}", edgeString); titanGraph.removeEdge(edge); - LOG.info("Removed edge {}", edge); - } - - /** - * Return the edge and target vertex for the specified edge ID. - * - * @param edgeId - * @return edge and target vertex - */ - public Pair<Edge, Vertex> getEdgeAndTargetVertex(String edgeId) { - final Edge edge = titanGraph.getEdge(edgeId); - Vertex referredVertex = edge.getVertex(Direction.IN); - return Pair.of(edge, referredVertex); + LOG.info("Removed {}", edgeString); } /** @@ -216,27 +279,22 @@ public final class GraphHelper { * @param vertex */ public void removeVertex(Vertex vertex) { - LOG.debug("Removing vertex {}", vertex); + String vertexString = string(vertex); + LOG.debug("Removing {}", vertexString); titanGraph.removeVertex(vertex); - LOG.info("Removed vertex {}", vertex); + LOG.info("Removed {}", vertexString); } public Vertex getVertexForGUID(String guid) throws EntityNotFoundException { - return getVertexForProperty(Constants.GUID_PROPERTY_KEY, guid); + return findVertex(Constants.GUID_PROPERTY_KEY, guid); } - public Vertex getVertexForProperty(String propertyKey, Object value) throws EntityNotFoundException { - Vertex instanceVertex = findVertex(propertyKey, value); - if (instanceVertex == null) { - LOG.debug("Could not find a vertex with {}={}", propertyKey, value); - throw new EntityNotFoundException("Could not find an entity in the repository with " + propertyKey + "=" - + value); - } else { - LOG.debug("Found a vertex {} with {}={}", instanceVertex, propertyKey, value); - } + return findVertex(propertyKey, value, Constants.STATE_PROPERTY_KEY, Id.EntityState.ACTIVE.name()); + } - return instanceVertex; + public static String getQualifiedNameForMapKey(String prefix, String key) { + return prefix + "." + key; } public static String getQualifiedFieldName(ITypedInstance typedInstance, AttributeInfo attributeInfo) throws AtlasException { @@ -277,6 +335,10 @@ public final class GraphHelper { vertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), dataTypeName); } + public static String getIdFromVertex(Vertex vertex) { + return vertex.<String>getProperty(Constants.GUID_PROPERTY_KEY); + } + public static String getTypeName(Vertex instanceVertex) { return instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY); } @@ -292,7 +354,7 @@ public final class GraphHelper { */ public Vertex getVertexForInstanceByUniqueAttribute(ClassType classType, IReferenceableInstance instance) throws AtlasException { - LOG.debug("Checking if there is an instance with the same unique attributes for instance {}", instance); + LOG.debug("Checking if there is an instance with the same unique attributes for instance {}", instance.toShortString()); Vertex result = null; for (AttributeInfo attributeInfo : classType.fieldMapping().fields.values()) { if (attributeInfo.isUnique) { @@ -322,4 +384,18 @@ public final class GraphHelper { } LOG.debug("*******************Graph Dump****************************"); } + + public static String string(ITypedReferenceableInstance instance) { + return String.format("entity[type=%s guid=%]", instance.getTypeName(), instance.getId()._getId()); + } + + public static String string(Vertex vertex) { + return String.format("vertex[id=%s type=%s guid=%s]", vertex.getId().toString(), getTypeName(vertex), + getIdFromVertex(vertex)); + } + + public static String string(Edge edge) { + return String.format("edge[id=%s label=%s from %s -> to %s]", edge.getId().toString(), edge.getLabel(), + string(edge.getVertex(Direction.OUT)), string(edge.getVertex(Direction.IN))); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/40ee9492/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java index 1d682bb..df28ab3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java @@ -17,6 +17,7 @@ */ package org.apache.atlas.repository.graph; +import com.google.inject.Singleton; import com.thinkaurelius.titan.core.TitanGraph; import com.tinkerpop.blueprints.Direction; import com.tinkerpop.blueprints.Edge; @@ -43,15 +44,19 @@ import java.math.BigInteger; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; +import static org.apache.atlas.repository.graph.GraphHelper.string; + +@Singleton public final class GraphToTypedInstanceMapper { private static final Logger LOG = LoggerFactory.getLogger(GraphToTypedInstanceMapper.class); private static TypeSystem typeSystem = TypeSystem.getInstance(); - private final TitanGraph titanGraph; + private static final GraphHelper graphHelper = GraphHelper.getInstance(); + + private TitanGraph titanGraph; public GraphToTypedInstanceMapper(TitanGraph titanGraph) { this.titanGraph = titanGraph; @@ -97,12 +102,12 @@ public final class GraphToTypedInstanceMapper { } } - private void mapVertexToAttribute(Vertex instanceVertex, ITypedInstance typedInstance, AttributeInfo attributeInfo) throws AtlasException { LOG.debug("Mapping attributeInfo {}", attributeInfo.name); final IDataType dataType = attributeInfo.dataType(); final String vertexPropertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo); + String relationshipLabel = GraphHelper.getEdgeLabel(typedInstance, attributeInfo); switch (dataType.getTypeCategory()) { case PRIMITIVE: @@ -128,7 +133,9 @@ public final class GraphToTypedInstanceMapper { break; case STRUCT: - mapVertexToStructInstance(instanceVertex, typedInstance, attributeInfo); + ITypedStruct structInstance = mapVertexToStructInstance(instanceVertex, + (StructType) attributeInfo.dataType(), relationshipLabel, null); + typedInstance.set(attributeInfo.name, structInstance); break; case TRAIT: @@ -136,9 +143,8 @@ public final class GraphToTypedInstanceMapper { break; case CLASS: - String relationshipLabel = GraphHelper.getEdgeLabel(typedInstance, attributeInfo); Object idOrInstance = mapVertexToClassReference(instanceVertex, attributeInfo, relationshipLabel, - attributeInfo.dataType()); + attributeInfo.dataType(), null); if (idOrInstance != null) { typedInstance.set(attributeInfo.name, idOrInstance); } @@ -150,25 +156,30 @@ public final class GraphToTypedInstanceMapper { } private Object mapVertexToClassReference(Vertex instanceVertex, AttributeInfo attributeInfo, - String relationshipLabel, IDataType dataType) throws AtlasException { + String relationshipLabel, IDataType dataType, String edgeId) throws AtlasException { LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel); - Iterator<Edge> results = instanceVertex.getEdges(Direction.OUT, relationshipLabel).iterator(); - if (results.hasNext()) { - final Vertex referenceVertex = results.next().getVertex(Direction.IN); - if (referenceVertex != null) { - final String guid = referenceVertex.getProperty(Constants.GUID_PROPERTY_KEY); - LOG.debug("Found vertex {} for label {} with guid {}", referenceVertex, relationshipLabel, guid); - if (attributeInfo.isComposite) { - //Also, when you retrieve a type's instance, you get the complete object graph of the composites - LOG.debug("Found composite, mapping vertex to instance"); - return mapGraphToTypedInstance(guid, referenceVertex); - } else { - Id referenceId = + + Edge edge; + if (edgeId == null) { + edge = GraphHelper.getEdgeForLabel(instanceVertex, relationshipLabel);; + } else { + edge = graphHelper.getEdgeById(edgeId); + } + + if (edge != null) { + final Vertex referenceVertex = edge.getVertex(Direction.IN); + final String guid = referenceVertex.getProperty(Constants.GUID_PROPERTY_KEY); + LOG.debug("Found vertex {} for label {} with guid {}", referenceVertex, relationshipLabel, guid); + if (attributeInfo.isComposite) { + //Also, when you retrieve a type's instance, you get the complete object graph of the composites + LOG.debug("Found composite, mapping vertex to instance"); + return mapGraphToTypedInstance(guid, referenceVertex); + } else { + Id referenceId = new Id(guid, referenceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), - dataType.getName()); - LOG.debug("Found non-composite, adding id {} ", referenceId); - return referenceId; - } + dataType.getName()); + LOG.debug("Found non-composite, adding id {} ", referenceId); + return referenceId; } } @@ -212,8 +223,7 @@ public final class GraphToTypedInstanceMapper { break; case STRUCT: - return getStructInstanceFromVertex(instanceVertex, elementType, attributeInfo.name, edgeLabel, - (String) value); + return mapVertexToStructInstance(instanceVertex, (StructType) elementType, edgeLabel, (String) value); case CLASS: return mapVertexToClassReference(instanceVertex, attributeInfo, edgeLabel, elementType, (String) value); @@ -252,83 +262,27 @@ public final class GraphToTypedInstanceMapper { } } - private ITypedStruct getStructInstanceFromVertex(Vertex instanceVertex, IDataType elemType, - String attributeName, String relationshipLabel, String edgeId) throws AtlasException { - LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel); - for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) { - if (edgeId.equals(String.valueOf(edge.getId()))) { - Vertex structInstanceVertex = edge.getVertex(Direction.IN); - LOG.debug("mapping vertex {} to struct {}", structInstanceVertex, attributeName); - - if (structInstanceVertex != null) { - LOG.debug("Found struct instance vertex {}, mapping to instance {} ", structInstanceVertex, - elemType.getName()); - StructType structType = typeSystem.getDataType(StructType.class, elemType.getName()); - ITypedStruct structInstance = structType.createInstance(); - mapVertexToInstance(structInstanceVertex, structInstance, structType.fieldMapping().fields); - return structInstance; - } - - break; - } - } - - return null; - } - - private Object mapVertexToClassReference(Vertex instanceVertex, AttributeInfo attributeInfo, - String relationshipLabel, IDataType dataType, String edgeId) throws AtlasException { - LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel); - for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) { - if (edgeId.equals(String.valueOf(edge.getId()))) { - final Vertex referenceVertex = edge.getVertex(Direction.IN); - if (referenceVertex != null) { - final String guid = referenceVertex.getProperty(Constants.GUID_PROPERTY_KEY); - LOG.debug("Found vertex {} for label {} with guid {}", referenceVertex, relationshipLabel, - guid); - if (attributeInfo.isComposite) { - //Also, when you retrieve a type's instance, you get the complete object graph of the composites - LOG.debug("Found composite, mapping vertex to instance"); - return mapGraphToTypedInstance(guid, referenceVertex); - } else { - Id referenceId = - new Id(guid, referenceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), - dataType.getName()); - LOG.debug("Found non-composite, adding id {} ", referenceId); - return referenceId; - } - } - - break; - } - } - - return null; - } - - private void mapVertexToStructInstance(Vertex instanceVertex, ITypedInstance typedInstance, - AttributeInfo attributeInfo) throws AtlasException { - LOG.debug("mapping vertex {} to struct {}", instanceVertex, attributeInfo.name); - StructType structType = typeSystem.getDataType(StructType.class, attributeInfo.dataType().getName()); + private ITypedStruct mapVertexToStructInstance(Vertex instanceVertex, StructType structType, + String relationshipLabel, String edgeId) throws AtlasException { + LOG.debug("mapping {} to struct {}", string(instanceVertex), relationshipLabel); ITypedStruct structInstance = null; - String relationshipLabel = GraphHelper.getEdgeLabel(typedInstance, attributeInfo); - LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel); - final Iterable<Edge> edges = instanceVertex.getEdges(Direction.OUT, relationshipLabel); - if (edges.iterator().hasNext()) { - structInstance = structType.createInstance(); - typedInstance.set(attributeInfo.name, structInstance); + Edge edge; + if (edgeId == null) { + edge = GraphHelper.getEdgeForLabel(instanceVertex, relationshipLabel); + } else { + edge = graphHelper.getEdgeById(edgeId); } - for (Edge edge : edges) { - final Vertex structInstanceVertex = edge.getVertex(Direction.IN); - if (structInstanceVertex != null) { - LOG.debug("Found struct instance vertex {}, mapping to instance {} ", structInstanceVertex, + if (edge != null) { + structInstance = structType.createInstance(); + Vertex structInstanceVertex = edge.getVertex(Direction.IN); + LOG.debug("Found struct instance {}, mapping to instance {} ", string(structInstanceVertex), structInstance.getTypeName()); - mapVertexToInstance(structInstanceVertex, structInstance, structType.fieldMapping().fields); - break; - } + mapVertexToInstance(structInstanceVertex, structInstance, structType.fieldMapping().fields); + } + return structInstance; } private void mapVertexToTraitInstance(Vertex instanceVertex, ITypedReferenceableInstance typedInstance, http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/40ee9492/repository/src/main/java/org/apache/atlas/repository/graph/HardDeleteHandler.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/HardDeleteHandler.java b/repository/src/main/java/org/apache/atlas/repository/graph/HardDeleteHandler.java new file mode 100644 index 0000000..f8bbf73 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/graph/HardDeleteHandler.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.graph; + +import com.google.inject.Inject; +import com.tinkerpop.blueprints.Edge; +import com.tinkerpop.blueprints.Vertex; +import org.apache.atlas.AtlasException; +import org.apache.atlas.typesystem.types.TypeSystem; + +public class HardDeleteHandler extends DeleteHandler { + + private static final GraphHelper graphHelper = GraphHelper.getInstance(); + + @Inject + public HardDeleteHandler(TypeSystem typeSystem) { + super(typeSystem, true); + } + + @Override + protected void _deleteVertex(Vertex instanceVertex) { + graphHelper.removeVertex(instanceVertex); + } + + @Override + protected void deleteEdge(Edge edge) throws AtlasException { + graphHelper.removeEdge(edge); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/40ee9492/repository/src/main/java/org/apache/atlas/repository/graph/SoftDeleteHandler.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/SoftDeleteHandler.java b/repository/src/main/java/org/apache/atlas/repository/graph/SoftDeleteHandler.java new file mode 100644 index 0000000..aa78582 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/graph/SoftDeleteHandler.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.graph; + +import com.google.inject.Inject; +import com.tinkerpop.blueprints.Edge; +import com.tinkerpop.blueprints.Vertex; +import org.apache.atlas.AtlasException; +import org.apache.atlas.RequestContext; +import org.apache.atlas.typesystem.persistence.Id; +import org.apache.atlas.typesystem.types.TypeSystem; + +import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY; +import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY; + +public class SoftDeleteHandler extends DeleteHandler { + @Inject + public SoftDeleteHandler(TypeSystem typeSystem) { + super(typeSystem, false); + } + + @Override + protected void _deleteVertex(Vertex instanceVertex) { + Id.EntityState state = Id.EntityState.valueOf((String) instanceVertex.getProperty(STATE_PROPERTY_KEY)); + if (state != Id.EntityState.DELETED) { + GraphHelper.setProperty(instanceVertex, STATE_PROPERTY_KEY, Id.EntityState.DELETED.name()); + GraphHelper.setProperty(instanceVertex, MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime()); + } + } + + @Override + protected void deleteEdge(Edge edge) throws AtlasException { + Id.EntityState state = Id.EntityState.valueOf((String) edge.getProperty(STATE_PROPERTY_KEY)); + if (state != Id.EntityState.DELETED) { + GraphHelper.setProperty(edge, STATE_PROPERTY_KEY, Id.EntityState.DELETED.name()); + GraphHelper.setProperty(edge, MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime()); + } + } +}
