http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java b/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java deleted file mode 100644 index f0fef1f..0000000 --- a/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java +++ /dev/null @@ -1,468 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.repository.graph; - -import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX; -import static org.apache.atlas.repository.graph.GraphHelper.string; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Set; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.RequestContext; -import org.apache.atlas.repository.Constants; -import org.apache.atlas.repository.graph.GraphHelper.VertexInfo; -import org.apache.atlas.repository.graphdb.AtlasEdge; -import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; -import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.typesystem.exception.NullRequiredAttributeException; -import org.apache.atlas.typesystem.persistence.Id; -import org.apache.atlas.typesystem.types.AttributeInfo; -import org.apache.atlas.typesystem.types.DataTypes; -import org.apache.atlas.typesystem.types.FieldMapping; -import org.apache.atlas.typesystem.types.HierarchicalType; -import org.apache.atlas.typesystem.types.IDataType; -import org.apache.atlas.typesystem.types.StructType; -import org.apache.atlas.typesystem.types.TypeSystem; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public abstract class DeleteHandler { - public static final Logger LOG = LoggerFactory.getLogger(DeleteHandler.class); - - protected static final GraphHelper graphHelper = GraphHelper.getInstance(); - - protected TypeSystem typeSystem; - private boolean shouldUpdateReverseAttribute; - private boolean softDelete; - - public DeleteHandler(TypeSystem typeSystem, boolean shouldUpdateReverseAttribute, boolean softDelete) { - this.typeSystem = typeSystem; - this.shouldUpdateReverseAttribute = shouldUpdateReverseAttribute; - this.softDelete = softDelete; - } - - /** - * Deletes the specified entity vertices. - * Deletes any traits, composite entities, and structs owned by each entity. - * Also deletes all the references from/to the entity. - * - * @param instanceVertices - * @throws AtlasException - */ - public void deleteEntities(Collection<AtlasVertex> instanceVertices) throws AtlasException { - RequestContext requestContext = RequestContext.get(); - - Set<AtlasVertex> deletionCandidateVertices = new HashSet<>(); - - for (AtlasVertex instanceVertex : instanceVertices) { - String guid = GraphHelper.getGuid(instanceVertex); - Id.EntityState state = GraphHelper.getState(instanceVertex); - if (requestContext.getDeletedEntityIds().contains(guid) || state == Id.EntityState.DELETED) { - if (LOG.isDebugEnabled()) { - LOG.debug("Skipping deletion of {} as it is already deleted", guid); - } - - continue; - } - - // Get GUIDs and vertices for all deletion candidates. - Set<VertexInfo> compositeVertices = graphHelper.getCompositeVertices(instanceVertex); - - // Record all deletion candidate GUIDs in RequestContext - // and gather deletion candidate vertices. - for (VertexInfo vertexInfo : compositeVertices) { - requestContext.recordEntityDelete(vertexInfo.getGuid(), vertexInfo.getTypeName()); - deletionCandidateVertices.add(vertexInfo.getVertex()); - } - } - - // Delete traits and vertices. - for (AtlasVertex deletionCandidateVertex : deletionCandidateVertices) { - deleteAllTraits(deletionCandidateVertex); - deleteTypeVertex(deletionCandidateVertex, false); - } - } - - protected abstract void deleteEdge(AtlasEdge edge, boolean force) throws AtlasException; - - /** - * Deletes a type vertex - can be entity(class type) or just vertex(struct/trait type) - * @param instanceVertex - * @param typeCategory - * @throws AtlasException - */ - protected void deleteTypeVertex(AtlasVertex instanceVertex, DataTypes.TypeCategory typeCategory, boolean force) throws AtlasException { - switch (typeCategory) { - case STRUCT: - case TRAIT: - deleteTypeVertex(instanceVertex, force); - break; - - case CLASS: - deleteEntities(Collections.singletonList(instanceVertex)); - break; - - default: - throw new IllegalStateException("Type category " + typeCategory + " not handled"); - } - } - - /** - * Deleting any type vertex. Goes over the complex attributes and removes the references - * @param instanceVertex - * @throws AtlasException - */ - protected void deleteTypeVertex(AtlasVertex instanceVertex, boolean force) throws AtlasException { - if (LOG.isDebugEnabled()) { - LOG.debug("Deleting {}", string(instanceVertex)); - } - - String typeName = GraphHelper.getTypeName(instanceVertex); - IDataType type = typeSystem.getDataType(IDataType.class, typeName); - FieldMapping fieldMapping = getFieldMapping(type); - - for (AttributeInfo attributeInfo : fieldMapping.fields.values()) { - if (LOG.isDebugEnabled()) { - LOG.debug("Deleting attribute {} for {}", attributeInfo.name, string(instanceVertex)); - } - - String edgeLabel = GraphHelper.getEdgeLabel(type, attributeInfo); - - switch (attributeInfo.dataType().getTypeCategory()) { - case CLASS: - //If its class attribute, delete the reference - deleteEdgeReference(instanceVertex, edgeLabel, DataTypes.TypeCategory.CLASS, attributeInfo.isComposite); - break; - - case STRUCT: - //If its struct attribute, delete the reference - deleteEdgeReference(instanceVertex, edgeLabel, DataTypes.TypeCategory.STRUCT, false); - break; - - case ARRAY: - //For array attribute, if the element is struct/class, delete all the references - IDataType elementType = ((DataTypes.ArrayType) attributeInfo.dataType()).getElemType(); - DataTypes.TypeCategory elementTypeCategory = elementType.getTypeCategory(); - if (elementTypeCategory == DataTypes.TypeCategory.STRUCT || - elementTypeCategory == DataTypes.TypeCategory.CLASS) { - Iterator<AtlasEdge> edges = graphHelper.getOutGoingEdgesByLabel(instanceVertex, edgeLabel); - if (edges != null) { - while (edges.hasNext()) { - AtlasEdge edge = edges.next(); - deleteEdgeReference(edge, elementType.getTypeCategory(), attributeInfo.isComposite, false); - } - } - } - break; - - case MAP: - //For map attribute, if the value type is struct/class, delete all the references - DataTypes.MapType mapType = (DataTypes.MapType) attributeInfo.dataType(); - DataTypes.TypeCategory valueTypeCategory = mapType.getValueType().getTypeCategory(); - String propertyName = GraphHelper.getQualifiedFieldName(type, attributeInfo.name); - - if (valueTypeCategory == DataTypes.TypeCategory.STRUCT || - valueTypeCategory == DataTypes.TypeCategory.CLASS) { - List<String> keys = GraphHelper.getListProperty(instanceVertex, propertyName); - if (keys != null) { - for (String key : keys) { - String mapEdgeLabel = GraphHelper.getQualifiedNameForMapKey(edgeLabel, key); - deleteEdgeReference(instanceVertex, mapEdgeLabel, valueTypeCategory, attributeInfo.isComposite); - } - } - } - } - } - - deleteVertex(instanceVertex, force); - } - - /** - * Force delete is used to remove struct/trait in case of entity updates - * @param edge - * @param typeCategory - * @param isComposite - * @param forceDeleteStructTrait - * @return returns true if the edge reference is hard deleted - * @throws AtlasException - */ - public boolean deleteEdgeReference(AtlasEdge edge, DataTypes.TypeCategory typeCategory, boolean isComposite, - boolean forceDeleteStructTrait) throws AtlasException { - if (LOG.isDebugEnabled()) { - LOG.debug("Deleting {}", string(edge)); - } - - boolean forceDelete = - (typeCategory == DataTypes.TypeCategory.STRUCT || typeCategory == DataTypes.TypeCategory.TRAIT) && forceDeleteStructTrait; - if (typeCategory == DataTypes.TypeCategory.STRUCT || typeCategory == DataTypes.TypeCategory.TRAIT - || (typeCategory == DataTypes.TypeCategory.CLASS && isComposite)) { - //If the vertex is of type struct/trait, delete the edge and then the reference vertex as the vertex is not shared by any other entities. - //If the vertex is of type class, and its composite attribute, this reference vertex' lifecycle is controlled - //through this delete, hence delete the edge and the reference vertex. - AtlasVertex vertexForDelete = edge.getInVertex(); - - //If deleting the edge and then the in vertex, reverse attribute shouldn't be updated - deleteEdge(edge, false, forceDelete); - deleteTypeVertex(vertexForDelete, typeCategory, forceDelete); - } else { - //If the vertex is of type class, and its not a composite attributes, the reference AtlasVertex' lifecycle is not controlled - //through this delete. Hence just remove the reference edge. Leave the reference AtlasVertex as is - - //If deleting just the edge, reverse attribute should be updated for any references - //For example, for the department type system, if the person's manager edge is deleted, subordinates of manager should be updated - deleteEdge(edge, true, false); - } - return !softDelete || forceDelete; - } - - public void deleteEdgeReference(AtlasVertex outVertex, String edgeLabel, DataTypes.TypeCategory typeCategory, - boolean isComposite) throws AtlasException { - AtlasEdge edge = graphHelper.getEdgeForLabel(outVertex, edgeLabel); - if (edge != null) { - deleteEdgeReference(edge, typeCategory, isComposite, false); - } - } - - protected void deleteEdge(AtlasEdge edge, boolean updateReverseAttribute, boolean force) throws AtlasException { - //update reverse attribute - if (updateReverseAttribute) { - AttributeInfo attributeInfo = getAttributeForEdge(edge.getLabel()); - if (attributeInfo.reverseAttributeName != null) { - deleteEdgeBetweenVertices(edge.getInVertex(), edge.getOutVertex(), - attributeInfo.reverseAttributeName); - } - } - - deleteEdge(edge, force); - } - - protected void deleteVertex(AtlasVertex instanceVertex, boolean force) throws AtlasException { - //Update external references(incoming edges) to this vertex - if (LOG.isDebugEnabled()) { - LOG.debug("Setting the external references to {} to null(removing edges)", string(instanceVertex)); - } - - for (AtlasEdge edge : (Iterable<AtlasEdge>) instanceVertex.getEdges(AtlasEdgeDirection.IN)) { - Id.EntityState edgeState = GraphHelper.getState(edge); - if (edgeState == Id.EntityState.ACTIVE) { - //Delete only the active edge references - AttributeInfo attribute = getAttributeForEdge(edge.getLabel()); - //TODO use delete edge instead?? - deleteEdgeBetweenVertices(edge.getOutVertex(), edge.getInVertex(), attribute.name); - } - } - _deleteVertex(instanceVertex, force); - } - - protected abstract void _deleteVertex(AtlasVertex instanceVertex, boolean force); - - /** - * Deletes the edge between outvertex and inVertex. The edge is for attribute attributeName of outVertex - * @param outVertex - * @param inVertex - * @param attributeName - * @throws AtlasException - */ - protected void deleteEdgeBetweenVertices(AtlasVertex outVertex, AtlasVertex inVertex, String attributeName) throws AtlasException { - if (LOG.isDebugEnabled()) { - LOG.debug("Removing edge from {} to {} with attribute name {}", string(outVertex), string(inVertex), - attributeName); - } - - String typeName = GraphHelper.getTypeName(outVertex); - String outId = GraphHelper.getGuid(outVertex); - Id.EntityState state = GraphHelper.getState(outVertex); - if ((outId != null && RequestContext.get().isDeletedEntity(outId)) || state == Id.EntityState.DELETED) { - //If the reference vertex is marked for deletion, skip updating the reference - return; - } - - IDataType type = typeSystem.getDataType(IDataType.class, typeName); - AttributeInfo attributeInfo = getFieldMapping(type).fields.get(attributeName); - String propertyName = GraphHelper.getQualifiedFieldName(type, attributeName); - String edgeLabel = EDGE_LABEL_PREFIX + propertyName; - AtlasEdge edge = null; - - switch (attributeInfo.dataType().getTypeCategory()) { - case CLASS: - //If its class attribute, its the only edge between two vertices - if (attributeInfo.multiplicity.nullAllowed()) { - edge = graphHelper.getEdgeForLabel(outVertex, edgeLabel); - if (shouldUpdateReverseAttribute) { - GraphHelper.setProperty(outVertex, propertyName, null); - } - } else { - // Cannot unset a required attribute. - throw new NullRequiredAttributeException("Cannot unset required attribute " + GraphHelper.getQualifiedFieldName(type, attributeName) + - " on " + GraphHelper.getVertexDetails(outVertex) + " edge = " + edgeLabel); - } - break; - - case ARRAY: - //If its array attribute, find the right edge between the two vertices and update array property - List<String> elements = GraphHelper.getListProperty(outVertex, propertyName); - if (elements != null) { - elements = new ArrayList<>(elements); //Make a copy, else list.remove reflects on titan.getProperty() - for (String elementEdgeId : elements) { - AtlasEdge elementEdge = graphHelper.getEdgeByEdgeId(outVertex, edgeLabel, elementEdgeId); - if (elementEdge == null) { - continue; - } - - AtlasVertex elementVertex = elementEdge.getInVertex(); - if (elementVertex.equals(inVertex)) { - edge = elementEdge; - - //TODO element.size includes deleted items as well. should exclude - if (!attributeInfo.multiplicity.nullAllowed() - && elements.size() <= attributeInfo.multiplicity.lower) { - // Deleting this edge would violate the attribute's lower bound. - throw new NullRequiredAttributeException( - "Cannot remove array element from required attribute " + - GraphHelper.getQualifiedFieldName(type, attributeName) + " on " - + GraphHelper.getVertexDetails(outVertex) + " " + GraphHelper.getEdgeDetails(elementEdge)); - } - - if (shouldUpdateReverseAttribute) { - //if composite attribute, remove the reference as well. else, just remove the edge - //for example, when table is deleted, process still references the table - //but when column is deleted, table will not reference the deleted column - if (LOG.isDebugEnabled()) { - LOG.debug("Removing edge {} from the array attribute {}", string(elementEdge), - attributeName); - } - - // Remove all occurrences of the edge ID from the list. - // This prevents dangling edge IDs (i.e. edge IDs for deleted edges) - // from the remaining in the list if there are duplicates. - elements.removeAll(Collections.singletonList(elementEdge.getId().toString())); - GraphHelper.setProperty(outVertex, propertyName, elements); - break; - - } - } - } - } - break; - - case MAP: - //If its map attribute, find the right edge between two vertices and update map property - List<String> keys = GraphHelper.getListProperty(outVertex, propertyName); - if (keys != null) { - keys = new ArrayList<>(keys); //Make a copy, else list.remove reflects on titan.getProperty() - for (String key : keys) { - String keyPropertyName = GraphHelper.getQualifiedNameForMapKey(propertyName, key); - String mapEdgeId = GraphHelper.getSingleValuedProperty(outVertex, keyPropertyName, String.class); - AtlasEdge mapEdge = graphHelper.getEdgeByEdgeId(outVertex, keyPropertyName, mapEdgeId); - if(mapEdge != null) { - AtlasVertex mapVertex = mapEdge.getInVertex(); - if (mapVertex.getId().toString().equals(inVertex.getId().toString())) { - //TODO keys.size includes deleted items as well. should exclude - if (attributeInfo.multiplicity.nullAllowed() || keys.size() > attributeInfo.multiplicity.lower) { - edge = mapEdge; - } else { - // Deleting this entry would violate the attribute's lower bound. - throw new NullRequiredAttributeException( - "Cannot remove map entry " + keyPropertyName + " from required attribute " + - GraphHelper.getQualifiedFieldName(type, attributeName) + " on " + GraphHelper.getVertexDetails(outVertex) + " " + GraphHelper.getEdgeDetails(mapEdge)); - } - - if (shouldUpdateReverseAttribute) { - //remove this key - if (LOG.isDebugEnabled()) { - LOG.debug("Removing edge {}, key {} from the map attribute {}", string(mapEdge), key, - attributeName); - } - - keys.remove(key); - GraphHelper.setProperty(outVertex, propertyName, keys); - GraphHelper.setProperty(outVertex, keyPropertyName, null); - } - break; - } - } - } - } - break; - - case STRUCT: - case TRAIT: - break; - - default: - throw new IllegalStateException("There can't be an edge from " + GraphHelper.getVertexDetails(outVertex) + " to " - + GraphHelper.getVertexDetails(inVertex) + " with attribute name " + attributeName + " which is not class/array/map attribute"); - } - - if (edge != null) { - deleteEdge(edge, false); - RequestContext requestContext = RequestContext.get(); - GraphHelper.setProperty(outVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, - requestContext.getRequestTime()); - GraphHelper.setProperty(outVertex, Constants.MODIFIED_BY_KEY, requestContext.getUser()); - requestContext.recordEntityUpdate(outId); - } - } - - protected AttributeInfo getAttributeForEdge(String edgLabel) throws AtlasException { - AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(edgLabel); - IDataType referenceType = typeSystem.getDataType(IDataType.class, atlasEdgeLabel.getTypeName()); - return getFieldMapping(referenceType).fields.get(atlasEdgeLabel.getAttributeName()); - } - - protected FieldMapping getFieldMapping(IDataType type) { - switch (type.getTypeCategory()) { - case CLASS: - case TRAIT: - return ((HierarchicalType)type).fieldMapping(); - - case STRUCT: - return ((StructType)type).fieldMapping(); - - default: - throw new IllegalStateException("Type " + type + " doesn't have any fields!"); - } - } - - /** - * Delete all traits from the specified vertex. - * @param instanceVertex - * @throws AtlasException - */ - private void deleteAllTraits(AtlasVertex instanceVertex) throws AtlasException { - List<String> traitNames = GraphHelper.getTraitNames(instanceVertex); - - if (LOG.isDebugEnabled()) { - LOG.debug("Deleting traits {} for {}", traitNames, string(instanceVertex)); - } - - String typeName = GraphHelper.getTypeName(instanceVertex); - - for (String traitNameToBeDeleted : traitNames) { - String relationshipLabel = GraphHelper.getTraitLabel(typeName, traitNameToBeDeleted); - deleteEdgeReference(instanceVertex, relationshipLabel, DataTypes.TypeCategory.TRAIT, false); - } - } -}
http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/repository/src/main/java/org/apache/atlas/repository/graph/EntityProcessor.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/EntityProcessor.java b/repository/src/main/java/org/apache/atlas/repository/graph/EntityProcessor.java deleted file mode 100644 index 892b36d..0000000 --- a/repository/src/main/java/org/apache/atlas/repository/graph/EntityProcessor.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.repository.graph; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.repository.RepositoryException; -import org.apache.atlas.typesystem.IReferenceableInstance; -import org.apache.atlas.typesystem.ITypedReferenceableInstance; -import org.apache.atlas.typesystem.persistence.Id; -import org.apache.atlas.typesystem.types.DataTypes; -import org.apache.atlas.typesystem.types.ObjectGraphWalker; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.Map; - -@Deprecated -public final class EntityProcessor implements ObjectGraphWalker.NodeProcessor { - - private final Map<Id, IReferenceableInstance> idToInstanceMap; - - public EntityProcessor() { - idToInstanceMap = new LinkedHashMap<>(); - } - - public Collection<IReferenceableInstance> getInstances() { - ArrayList<IReferenceableInstance> instances = new ArrayList<>(idToInstanceMap.values()); - Collections.reverse(instances); - return instances; - } - - @Override - public void processNode(ObjectGraphWalker.Node nd) throws AtlasException { - IReferenceableInstance ref = null; - Id id = null; - - if (nd.attributeName == null) { - ref = (IReferenceableInstance) nd.instance; - id = ref.getId(); - } else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) { - if (nd.value != null && (nd.value instanceof Id)) { - id = (Id) nd.value; - } - } - - if (id != null) { - if (id.isUnassigned()) { - if (ref != null) { - if (idToInstanceMap.containsKey(id)) { // Oops - throw new RepositoryException( - String.format("Unexpected internal error: Id %s processed again", id)); - } - - idToInstanceMap.put(id, ref); - } - } - } - } - - public void addInstanceIfNotExists(ITypedReferenceableInstance ref) { - if(!idToInstanceMap.containsKey(ref.getId())) { - idToInstanceMap.put(ref.getId(), ref); - } - } -} http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java deleted file mode 100644 index 2e8ae0c..0000000 --- a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java +++ /dev/null @@ -1,174 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.repository.graph; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.RequestContext; -import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.typesystem.ITypedInstance; -import org.apache.atlas.typesystem.ITypedReferenceableInstance; -import org.apache.atlas.typesystem.persistence.Id; -import org.apache.atlas.typesystem.types.AttributeInfo; -import org.apache.atlas.typesystem.types.DataTypes; -import org.apache.atlas.typesystem.types.EnumValue; -import org.apache.atlas.typesystem.types.IDataType; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; -import java.util.Map; - -@Deprecated -public class FullTextMapper { - - private static final Logger LOG = LoggerFactory.getLogger(FullTextMapper.class); - - private final GraphToTypedInstanceMapper graphToTypedInstanceMapper; - private final TypedInstanceToGraphMapper typedInstanceToGraphMapper; - - private static final GraphHelper graphHelper = GraphHelper.getInstance(); - - private static final String FULL_TEXT_DELIMITER = " "; - - public FullTextMapper(TypedInstanceToGraphMapper typedInstanceToGraphMapper, - GraphToTypedInstanceMapper graphToTypedInstanceMapper) { - this.graphToTypedInstanceMapper = graphToTypedInstanceMapper; - this.typedInstanceToGraphMapper = typedInstanceToGraphMapper; - } - - public String mapRecursive(AtlasVertex instanceVertex, boolean followReferences) throws AtlasException { - String guid = GraphHelper.getGuid(instanceVertex); - ITypedReferenceableInstance typedReference; - RequestContext context = RequestContext.get(); - typedReference = context.getInstanceV1(guid); - if (typedReference != null) { - - if (LOG.isDebugEnabled()) { - LOG.debug("Cache hit: guid = {}, entityId = {}", guid, typedReference.getId()._getId()); - } - } else { - typedReference = - graphToTypedInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex); - context.cache(typedReference); - - if (LOG.isDebugEnabled()) { - LOG.debug("Cache miss: guid = {}, entityId = {}", guid, typedReference.getId().getId()); - } - } - String fullText = forInstance(typedReference, followReferences); - StringBuilder fullTextBuilder = - new StringBuilder(typedReference.getTypeName()).append(FULL_TEXT_DELIMITER).append(fullText); - - List<String> traits = typedReference.getTraits(); - for (String traitName : traits) { - String traitText = forInstance((ITypedInstance) typedReference.getTrait(traitName), false); - fullTextBuilder.append(FULL_TEXT_DELIMITER).append(traitName).append(FULL_TEXT_DELIMITER) - .append(traitText); - } - return fullTextBuilder.toString(); - } - - private String forAttribute(IDataType type, Object value, boolean followReferences) - throws AtlasException { - if (value == null) { - return null; - } - switch (type.getTypeCategory()) { - case PRIMITIVE: - return String.valueOf(value); - case ENUM: - - return ((EnumValue) value).value; - - case ARRAY: - StringBuilder fullText = new StringBuilder(); - IDataType elemType = ((DataTypes.ArrayType) type).getElemType(); - List list = (List) value; - - for (Object element : list) { - String elemFullText = forAttribute(elemType, element, false); - if (StringUtils.isNotEmpty(elemFullText)) { - fullText = fullText.append(FULL_TEXT_DELIMITER).append(elemFullText); - } - } - return fullText.toString(); - - case MAP: - fullText = new StringBuilder(); - IDataType keyType = ((DataTypes.MapType) type).getKeyType(); - IDataType valueType = ((DataTypes.MapType) type).getValueType(); - Map map = (Map) value; - - for (Object entryObj : map.entrySet()) { - Map.Entry entry = (Map.Entry) entryObj; - String keyFullText = forAttribute(keyType, entry.getKey(), false); - if (StringUtils.isNotEmpty(keyFullText)) { - fullText = fullText.append(FULL_TEXT_DELIMITER).append(keyFullText); - } - String valueFullText = forAttribute(valueType, entry.getValue(), false); - if (StringUtils.isNotEmpty(valueFullText)) { - fullText = fullText.append(FULL_TEXT_DELIMITER).append(valueFullText); - } - } - return fullText.toString(); - - case CLASS: - if (followReferences) { - Id refId = ((ITypedReferenceableInstance) value).getId(); - String refGuid = refId._getId(); - AtlasVertex refVertex = typedInstanceToGraphMapper.lookupVertex(refId); - if(refVertex == null) { - refVertex = graphHelper.getVertexForGUID(refGuid); - } - return mapRecursive(refVertex, false); - } - break; - - case STRUCT: - if (followReferences) { - return forInstance((ITypedInstance) value, true); - } - break; - - default: - throw new IllegalStateException("Unhandled type category " + type.getTypeCategory()); - - } - return null; - } - - private String forInstance(ITypedInstance typedInstance, boolean followReferences) - throws AtlasException { - StringBuilder fullText = new StringBuilder(); - for (AttributeInfo attributeInfo : typedInstance.fieldMapping().fields.values()) { - Object attrValue = typedInstance.get(attributeInfo.name); - if (attrValue == null) { - continue; - } - - String attrFullText = forAttribute(attributeInfo.dataType(), attrValue, followReferences); - if (StringUtils.isNotEmpty(attrFullText)) { - fullText = - fullText.append(FULL_TEXT_DELIMITER).append(attributeInfo.name).append(FULL_TEXT_DELIMITER) - .append(attrFullText); - } - } - return fullText.toString(); - } -} http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java index 76acf8c..c42aa15 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java @@ -17,7 +17,7 @@ */ package org.apache.atlas.repository.graph; -import org.apache.atlas.RequestContext; +import org.apache.atlas.RequestContextV1; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasClassification; import org.apache.atlas.model.instance.AtlasEntity; @@ -203,7 +203,7 @@ public class FullTextMapperV2 { } private AtlasEntityWithExtInfo getAndCacheEntity(String guid) throws AtlasBaseException { - RequestContext context = RequestContext.get(); + RequestContextV1 context = RequestContextV1.get(); AtlasEntityWithExtInfo entityWithExtInfo = context.getInstanceV2(guid); if (entityWithExtInfo == null) { http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java deleted file mode 100755 index 74886b5..0000000 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java +++ /dev/null @@ -1,505 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.repository.graph; - -import com.google.common.base.Preconditions; -import org.apache.atlas.AtlasException; -import org.apache.atlas.CreateUpdateEntitiesResult; -import org.apache.atlas.GraphTransactionInterceptor; -import org.apache.atlas.RequestContext; -import org.apache.atlas.annotation.GraphTransaction; -import org.apache.atlas.model.instance.GuidMapping; -import org.apache.atlas.model.legacy.EntityResult; -import org.apache.atlas.repository.Constants; -import org.apache.atlas.repository.MetadataRepository; -import org.apache.atlas.repository.RepositoryException; -import org.apache.atlas.repository.graphdb.AtlasEdge; -import org.apache.atlas.repository.graphdb.AtlasGraph; -import org.apache.atlas.repository.graphdb.AtlasGraphQuery; -import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.typesystem.ITypedReferenceableInstance; -import org.apache.atlas.typesystem.ITypedStruct; -import org.apache.atlas.typesystem.exception.EntityExistsException; -import org.apache.atlas.typesystem.exception.EntityNotFoundException; -import org.apache.atlas.typesystem.exception.TraitNotFoundException; -import org.apache.atlas.typesystem.persistence.Id; -import org.apache.atlas.typesystem.types.AttributeInfo; -import org.apache.atlas.typesystem.types.ClassType; -import org.apache.atlas.typesystem.types.DataTypes; -import org.apache.atlas.typesystem.types.IDataType; -import org.apache.atlas.typesystem.types.TypeSystem; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; - -import javax.inject.Inject; -import javax.inject.Singleton; -import java.util.*; - -/** - * An implementation backed by a Graph database provided - * as a Graph Service. - */ -@Singleton -@Component -@Deprecated -public class GraphBackedMetadataRepository implements MetadataRepository { - - private static final Logger LOG = LoggerFactory.getLogger(GraphBackedMetadataRepository.class); - - private static TypeSystem typeSystem = TypeSystem.getInstance(); - - private static final GraphHelper graphHelper = GraphHelper.getInstance(); - - private DeleteHandler deleteHandler; - - private final AtlasGraph atlasGraph; - private final GraphToTypedInstanceMapper graphToInstanceMapper; - - @Inject - public GraphBackedMetadataRepository(DeleteHandler deleteHandler, AtlasGraph atlasGraph) { - this.atlasGraph = atlasGraph; - this.graphToInstanceMapper = new GraphToTypedInstanceMapper(atlasGraph); - this.deleteHandler = deleteHandler; - } - - public GraphToTypedInstanceMapper getGraphToInstanceMapper() { - return graphToInstanceMapper; - } - - @Override - public String getTypeAttributeName() { - return Constants.ENTITY_TYPE_PROPERTY_KEY; - } - - @Override - public String getStateAttributeName() { - return Constants.STATE_PROPERTY_KEY; - } - - /** - * Returns the property key used to store super type names. - * - * @return property key used to store super type names. - */ - @Override - public String getSuperTypeAttributeName() { - return Constants.SUPER_TYPES_PROPERTY_KEY; - } - - public String getIdAttributeName() { - return Constants.GUID_PROPERTY_KEY; - } - - @Override - public String getVersionAttributeName() { - return Constants.VERSION_PROPERTY_KEY; - } - - @Override - public String getTraitLabel(IDataType<?> dataType, String traitName) { - return GraphHelper.getTraitLabel(dataType.getName(), traitName); - } - - @Override - public String getFieldNameInVertex(IDataType<?> dataType, AttributeInfo aInfo) throws AtlasException { - if (aInfo.name.startsWith(Constants.INTERNAL_PROPERTY_KEY_PREFIX)) { - return aInfo.name; - } - return GraphHelper.encodePropertyKey(GraphHelper.getQualifiedFieldName(dataType, aInfo.name)); - } - - public String getFieldNameInVertex(IDataType<?> dataType, String attrName) throws AtlasException { - return GraphHelper.getQualifiedFieldName(dataType, attrName); - } - - @Override - public String getEdgeLabel(IDataType<?> dataType, AttributeInfo aInfo) throws AtlasException { - return GraphHelper.getEdgeLabel(dataType, aInfo); - } - - @Override - @GraphTransaction - public CreateUpdateEntitiesResult createEntities(ITypedReferenceableInstance... entities) throws RepositoryException, - EntityExistsException { - if (LOG.isDebugEnabled()) { - LOG.debug("adding entities={}", entities); - } - - try { - TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper, deleteHandler); - instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.CREATE, entities); - List<String> createdGuids = RequestContext.get().getCreatedEntityIds(); - CreateUpdateEntitiesResult result = new CreateUpdateEntitiesResult(); - EntityResult entityResult = new EntityResult(createdGuids, null, null); - GuidMapping mapping = instanceToGraphMapper.createGuidMapping(); - result.setEntityResult(entityResult); - result.setGuidMapping(mapping); - return result; - } catch (EntityExistsException e) { - throw e; - } catch (AtlasException e) { - throw new RepositoryException(e); - } - } - - @Override - @GraphTransaction - public ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException, EntityNotFoundException { - return getEntityDefinitions(guid).get(0); - } - - @Override - @GraphTransaction - public List<ITypedReferenceableInstance> getEntityDefinitions(String... guids) throws RepositoryException, EntityNotFoundException { - if (LOG.isDebugEnabled()) { - LOG.debug("Retrieving entities with guids={}", Arrays.toString(guids)); - } - - RequestContext context = RequestContext.get(); - ITypedReferenceableInstance[] result = new ITypedReferenceableInstance[guids.length]; - - // Map of the guids of instances not in the cache to their index(es) in the result. - // This is used to put the loaded instances into the location(s) corresponding - // to their guid in the result. Note that a set is needed since guids can - // appear more than once in the list. - Map<String, Set<Integer>> uncachedGuids = new HashMap<>(); - - for (int i = 0; i < guids.length; i++) { - String guid = guids[i]; - - // First, check the cache. - ITypedReferenceableInstance cached = context.getInstanceV1(guid); - if (cached != null) { - result[i] = cached; - } else { - Set<Integer> indices = uncachedGuids.get(guid); - if (indices == null) { - indices = new HashSet<>(1); - uncachedGuids.put(guid, indices); - } - indices.add(i); - } - } - - List<String> guidsToFetch = new ArrayList<>(uncachedGuids.keySet()); - Map<String, AtlasVertex> instanceVertices = graphHelper.getVerticesForGUIDs(guidsToFetch); - - // search for missing entities - if (instanceVertices.size() != guidsToFetch.size()) { - Set<String> missingGuids = new HashSet<String>(guidsToFetch); - missingGuids.removeAll(instanceVertices.keySet()); - if (!missingGuids.isEmpty()) { - if (LOG.isDebugEnabled()) { - LOG.debug("Failed to find guids={}", missingGuids); - } - throw new EntityNotFoundException( - "Could not find entities in the repository with guids: " + missingGuids.toString()); - } - } - - for (String guid : guidsToFetch) { - try { - ITypedReferenceableInstance entity = graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertices.get(guid)); - for(int index : uncachedGuids.get(guid)) { - result[index] = entity; - } - } catch (AtlasException e) { - throw new RepositoryException(e); - } - } - return Arrays.asList(result); - } - - @Override - @GraphTransaction - public ITypedReferenceableInstance getEntityDefinition(String entityType, String attribute, Object value) - throws AtlasException { - if (LOG.isDebugEnabled()) { - LOG.debug("Retrieving entity with type={} and {}={}", entityType, attribute, value); - } - - IDataType type = typeSystem.getDataType(IDataType.class, entityType); - String propertyKey = getFieldNameInVertex(type, attribute); - AtlasVertex instanceVertex = graphHelper.findVertex(propertyKey, value, - Constants.ENTITY_TYPE_PROPERTY_KEY, entityType, - Constants.STATE_PROPERTY_KEY, Id.EntityState.ACTIVE.name()); - - String guid = GraphHelper.getGuid(instanceVertex); - ITypedReferenceableInstance cached = RequestContext.get().getInstanceV1(guid); - if(cached != null) { - return cached; - } - return graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex); - } - - @Override - @GraphTransaction - public List<String> getEntityList(String entityType) throws RepositoryException { - if (LOG.isDebugEnabled()) { - LOG.debug("Retrieving entity list for type={}", entityType); - } - - AtlasGraphQuery query = getGraph().query().has(Constants.ENTITY_TYPE_PROPERTY_KEY, entityType); - Iterator<AtlasVertex> results = query.vertices().iterator(); - if (!results.hasNext()) { - return Collections.emptyList(); - } - - ArrayList<String> entityList = new ArrayList<>(); - while (results.hasNext()) { - AtlasVertex vertex = results.next(); - entityList.add(GraphHelper.getGuid(vertex)); - } - - return entityList; - } - - /** - * Gets the list of trait names for a given entity represented by a guid. - * - * @param guid globally unique identifier for the entity - * @return a list of trait names for the given entity guid - * @throws RepositoryException - */ - @Override - @GraphTransaction - public List<String> getTraitNames(String guid) throws AtlasException { - if (LOG.isDebugEnabled()) { - LOG.debug("Retrieving trait names for entity={}", guid); - } - - AtlasVertex instanceVertex = graphHelper.getVertexForGUID(guid); - return GraphHelper.getTraitNames(instanceVertex); - } - - /** - * Adds a new trait to the list of entities represented by their respective guids - * @param entityGuids list of globally unique identifier for the entities - * @param traitInstance trait instance that needs to be added to entities - * @throws RepositoryException - */ - @Override - @GraphTransaction - public void addTrait(List<String> entityGuids, ITypedStruct traitInstance) throws RepositoryException { - Preconditions.checkNotNull(entityGuids, "entityGuids list cannot be null"); - Preconditions.checkNotNull(traitInstance, "Trait instance cannot be null"); - - if (LOG.isDebugEnabled()) { - LOG.debug("Adding a new trait={} for entities={}", traitInstance.getTypeName(), entityGuids); - } - - GraphTransactionInterceptor.lockObjectAndReleasePostCommit(entityGuids); - for (String entityGuid : entityGuids) { - addTraitImpl(entityGuid, traitInstance); - } - } - - /** - * Adds a new trait to an existing entity represented by a guid. - * - * @param guid globally unique identifier for the entity - * @param traitInstance trait instance that needs to be added to entity - * @throws RepositoryException - */ - @Override - @GraphTransaction - public void addTrait(String guid, ITypedStruct traitInstance) throws RepositoryException { - Preconditions.checkNotNull(guid, "guid cannot be null"); - Preconditions.checkNotNull(traitInstance, "Trait instance cannot be null"); - - GraphTransactionInterceptor.lockObjectAndReleasePostCommit(guid); - addTraitImpl(guid, traitInstance); - } - - private void addTraitImpl(String guid, ITypedStruct traitInstance) throws RepositoryException { - final String traitName = traitInstance.getTypeName(); - if (LOG.isDebugEnabled()) { - LOG.debug("Adding a new trait={} for entity={}", traitName, guid); - } - - try { - AtlasVertex instanceVertex = graphHelper.getVertexForGUID(guid); - - // add the trait instance as a new vertex - final String typeName = GraphHelper.getTypeName(instanceVertex); - - TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper, deleteHandler); - instanceToGraphMapper.mapTraitInstanceToVertex(traitInstance, - typeSystem.getDataType(ClassType.class, typeName), instanceVertex); - - - // update the traits in entity once adding trait instance is successful - GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName); - GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, - RequestContext.get().getRequestTime()); - GraphHelper.setProperty(instanceVertex, Constants.MODIFIED_BY_KEY, RequestContext.get().getUser()); - - } catch (RepositoryException e) { - throw e; - } catch (Exception e) { - throw new RepositoryException(e); - } - } - - /** - * Deletes a given trait from an existing entity represented by a guid. - * - * @param guid globally unique identifier for the entity - * @param traitNameToBeDeleted name of the trait - * @throws RepositoryException - */ - @Override - @GraphTransaction - public void deleteTrait(String guid, String traitNameToBeDeleted) throws TraitNotFoundException, EntityNotFoundException, RepositoryException { - LOG.debug("Deleting trait={} from entity={}", traitNameToBeDeleted, guid); - GraphTransactionInterceptor.lockObjectAndReleasePostCommit(guid); - - AtlasVertex instanceVertex = graphHelper.getVertexForGUID(guid); - - List<String> traitNames = GraphHelper.getTraitNames(instanceVertex); - if (!traitNames.contains(traitNameToBeDeleted)) { - throw new TraitNotFoundException( - "Could not find trait=" + traitNameToBeDeleted + " in the repository for entity: " + guid); - } - - try { - final String entityTypeName = GraphHelper.getTypeName(instanceVertex); - String relationshipLabel = GraphHelper.getTraitLabel(entityTypeName, traitNameToBeDeleted); - AtlasEdge edge = graphHelper.getEdgeForLabel(instanceVertex, relationshipLabel); - if(edge != null) { - deleteHandler.deleteEdgeReference(edge, DataTypes.TypeCategory.TRAIT, false, true); - } - - // update the traits in entity once trait removal is successful - traitNames.remove(traitNameToBeDeleted); - updateTraits(instanceVertex, traitNames); - } catch (Exception e) { - throw new RepositoryException(e); - } - } - - - private void updateTraits(AtlasVertex instanceVertex, List<String> traitNames) { - // remove the key - instanceVertex.removeProperty(Constants.TRAIT_NAMES_PROPERTY_KEY); - - // add it back again - for (String traitName : traitNames) { - GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName); - } - GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, - RequestContext.get().getRequestTime()); - GraphHelper.setProperty(instanceVertex, Constants.MODIFIED_BY_KEY, RequestContext.get().getUser()); - } - - @Override - @GraphTransaction - public CreateUpdateEntitiesResult updateEntities(ITypedReferenceableInstance... entitiesUpdated) throws RepositoryException { - if (LOG.isDebugEnabled()) { - LOG.debug("updating entity {}", entitiesUpdated); - } - - try { - TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper, deleteHandler); - instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.UPDATE_FULL, - entitiesUpdated); - CreateUpdateEntitiesResult result = new CreateUpdateEntitiesResult(); - RequestContext requestContext = RequestContext.get(); - result.setEntityResult(createEntityResultFromContext(requestContext)); - GuidMapping mapping = instanceToGraphMapper.createGuidMapping(); - result.setGuidMapping(mapping); - return result; - } catch (AtlasException e) { - throw new RepositoryException(e); - } - } - - @Override - @GraphTransaction - public CreateUpdateEntitiesResult updatePartial(ITypedReferenceableInstance entity) throws RepositoryException { - if (LOG.isDebugEnabled()) { - LOG.debug("updating entity {}", entity); - } - - try { - TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper, deleteHandler); - instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.UPDATE_PARTIAL, entity); - RequestContext requestContext = RequestContext.get(); - CreateUpdateEntitiesResult result = new CreateUpdateEntitiesResult(); - GuidMapping mapping = instanceToGraphMapper.createGuidMapping(); - result.setEntityResult(createEntityResultFromContext(requestContext)); - result.setGuidMapping(mapping); - return result; - } catch (AtlasException e) { - throw new RepositoryException(e); - } - } - - - - @Override - @GraphTransaction - public EntityResult deleteEntities(List<String> guids) throws RepositoryException { - - if (guids == null || guids.size() == 0) { - throw new IllegalArgumentException("guids must be non-null and non-empty"); - } - - // Retrieve vertices for requested guids. - Map<String, AtlasVertex> vertices = graphHelper.getVerticesForGUIDs(guids); - Collection<AtlasVertex> deletionCandidates = vertices.values(); - - if(LOG.isDebugEnabled()) { - for(String guid : guids) { - if(! vertices.containsKey(guid)) { - // Entity does not exist - treat as non-error, since the caller - // wanted to delete the entity and it's already gone. - LOG.debug("Deletion request ignored for non-existent entity with guid " + guid); - } - } - } - - if (deletionCandidates.isEmpty()) { - LOG.info("No deletion candidate entities were found for guids %s", guids); - return new EntityResult(Collections.<String>emptyList(), Collections.<String>emptyList(), Collections.<String>emptyList()); - } - - try { - deleteHandler.deleteEntities(deletionCandidates); - } - catch (AtlasException e) { - throw new RepositoryException(e); - } - - RequestContext requestContext = RequestContext.get(); - return createEntityResultFromContext(requestContext); - } - - private EntityResult createEntityResultFromContext(RequestContext requestContext) { - return new EntityResult( - requestContext.getCreatedEntityIds(), - requestContext.getUpdatedEntityIds(), - requestContext.getDeletedEntityIds()); - } - - public AtlasGraph getGraph() throws RepositoryException { - return atlasGraph; - } -} http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java index 6eee24b..9f1206c 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java @@ -34,7 +34,6 @@ import org.apache.atlas.model.typedef.AtlasEnumDef; import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.atlas.repository.Constants; -import org.apache.atlas.repository.IndexCreationException; import org.apache.atlas.repository.IndexException; import org.apache.atlas.repository.RepositoryException; import org.apache.atlas.repository.graphdb.AtlasCardinality; @@ -49,13 +48,6 @@ import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeUtil; -import org.apache.atlas.typesystem.types.AttributeInfo; -import org.apache.atlas.typesystem.types.ClassType; -import org.apache.atlas.typesystem.types.DataTypes; -import org.apache.atlas.typesystem.types.IDataType; -import org.apache.atlas.typesystem.types.Multiplicity; -import org.apache.atlas.typesystem.types.StructType; -import org.apache.atlas.typesystem.types.TraitType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.configuration.Configuration; import org.slf4j.Logger; @@ -66,11 +58,9 @@ import javax.inject.Inject; import java.math.BigDecimal; import java.math.BigInteger; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.*; @@ -80,8 +70,7 @@ import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.*; * Adds index for properties of a given type when its added before any instances are added. */ @Component -public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChangeHandler, - TypeDefChangeListener { +public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChangeHandler, TypeDefChangeListener { private static final Logger LOG = LoggerFactory.getLogger(GraphBackedSearchIndexer.class); @@ -118,6 +107,110 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang } /** + * Initialize global indices for Titan graph on server activation. + * + * Since the indices are shared state, we need to do this only from an active instance. + */ + @Override + public void instanceIsActive() throws AtlasException { + LOG.info("Reacting to active: initializing index"); + try { + initialize(); + } catch (RepositoryException | IndexException e) { + throw new AtlasException("Error in reacting to active on initialization", e); + } + } + + @Override + public void instanceIsPassive() { + LOG.info("Reacting to passive state: No action right now."); + } + + @Override + public int getHandlerOrder() { + return HandlerOrder.GRAPH_BACKED_SEARCH_INDEXER.getOrder(); + } + + @Override + public void onChange(ChangedTypeDefs changedTypeDefs) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("Processing changed typedefs {}", changedTypeDefs); + } + AtlasGraphManagement management = null; + try { + management = provider.get().getManagementSystem(); + + // Update index for newly created types + if (CollectionUtils.isNotEmpty(changedTypeDefs.getCreateTypeDefs())) { + for (AtlasBaseTypeDef typeDef : changedTypeDefs.getCreateTypeDefs()) { + updateIndexForTypeDef(management, typeDef); + } + } + + // Update index for updated types + if (CollectionUtils.isNotEmpty(changedTypeDefs.getUpdatedTypeDefs())) { + for (AtlasBaseTypeDef typeDef : changedTypeDefs.getUpdatedTypeDefs()) { + updateIndexForTypeDef(management, typeDef); + } + } + + // Invalidate the property key for deleted types + if (CollectionUtils.isNotEmpty(changedTypeDefs.getDeletedTypeDefs())) { + for (AtlasBaseTypeDef typeDef : changedTypeDefs.getDeletedTypeDefs()) { + cleanupIndices(management, typeDef); + } + } + + //Commit indexes + commit(management); + } catch (RepositoryException | IndexException e) { + LOG.error("Failed to update indexes for changed typedefs", e); + attemptRollback(changedTypeDefs, management); + } + + } + + public Set<String> getVertexIndexKeys() { + if (recomputeIndexedKeys) { + AtlasGraphManagement management = null; + + try { + management = provider.get().getManagementSystem(); + + if (management != null) { + AtlasGraphIndex vertexIndex = management.getGraphIndex(Constants.VERTEX_INDEX); + + if (vertexIndex != null) { + recomputeIndexedKeys = false; + + Set<String> indexKeys = new HashSet<>(); + + for (AtlasPropertyKey fieldKey : vertexIndex.getFieldKeys()) { + indexKeys.add(fieldKey.getName()); + } + + vertexIndexKeys = indexKeys; + } + + management.commit(); + } + } catch (Exception excp) { + LOG.error("getVertexIndexKeys(): failed to get indexedKeys from graph", excp); + + if (management != null) { + try { + management.rollback(); + } catch (Exception e) { + LOG.error("getVertexIndexKeys(): rollback failed", e); + } + } + } + } + + return vertexIndexKeys; + } + + /** * Initializes the indices for the graph - create indices for Global AtlasVertex Keys */ private void initialize() throws RepositoryException, IndexException { @@ -220,81 +313,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang true, true); } - /** - * This is upon adding a new type to Store. - * - * @param dataTypes data type - * @throws AtlasException - */ - @Override - public void onAdd(Collection<? extends IDataType> dataTypes) throws AtlasException { - AtlasGraphManagement management = provider.get().getManagementSystem(); - - for (IDataType dataType : dataTypes) { - if (LOG.isDebugEnabled()) { - LOG.debug("Creating indexes for type name={}, definition={}", dataType.getName(), dataType.getClass()); - } - - try { - addIndexForType(management, dataType); - LOG.info("Index creation for type {} complete", dataType.getName()); - } catch (Throwable throwable) { - LOG.error("Error creating index for type {}", dataType, throwable); - //Rollback indexes if any failure - rollback(management); - throw new IndexCreationException("Error while creating index for type " + dataType, throwable); - } - } - - //Commit indexes - commit(management); - } - - @Override - public void onChange(Collection<? extends IDataType> dataTypes) throws AtlasException { - onAdd(dataTypes); - } - - public Set<String> getVertexIndexKeys() { - if (recomputeIndexedKeys) { - AtlasGraphManagement management = null; - - try { - management = provider.get().getManagementSystem(); - - if (management != null) { - AtlasGraphIndex vertexIndex = management.getGraphIndex(Constants.VERTEX_INDEX); - - if (vertexIndex != null) { - recomputeIndexedKeys = false; - - Set<String> indexKeys = new HashSet<>(); - - for (AtlasPropertyKey fieldKey : vertexIndex.getFieldKeys()) { - indexKeys.add(fieldKey.getName()); - } - - vertexIndexKeys = indexKeys; - } - - management.commit(); - } - } catch (Exception excp) { - LOG.error("getVertexIndexKeys(): failed to get indexedKeys from graph", excp); - - if (management != null) { - try { - management.rollback(); - } catch (Exception e) { - LOG.error("getVertexIndexKeys(): rollback failed", e); - } - } - } - } - - return vertexIndexKeys; - } - private void addIndexForType(AtlasGraphManagement management, AtlasBaseTypeDef typeDef) { if (typeDef instanceof AtlasEnumDef) { // Only handle complex types like Struct, Classification and Entity @@ -414,82 +432,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang throw new IllegalArgumentException(String.format("Bad cardinality %s", cardinality)); } - private void addIndexForType(AtlasGraphManagement management, IDataType dataType) { - switch (dataType.getTypeCategory()) { - case PRIMITIVE: - case ENUM: - case ARRAY: - case MAP: - // do nothing since these are only attributes - // and not types like structs, traits or classes - break; - - case STRUCT: - StructType structType = (StructType) dataType; - createIndexForFields(management, structType, structType.fieldMapping().fields); - break; - - case TRAIT: - TraitType traitType = (TraitType) dataType; - createIndexForFields(management, traitType, traitType.fieldMapping().fields); - break; - - case CLASS: - ClassType classType = (ClassType) dataType; - createIndexForFields(management, classType, classType.fieldMapping().fields); - break; - - default: - throw new IllegalArgumentException("bad data type" + dataType); - } - } - - private void createIndexForFields(AtlasGraphManagement management, IDataType dataType, Map<String, AttributeInfo> fields) { - for (AttributeInfo field : fields.values()) { - createIndexForAttribute(management, dataType.getName(), field); - } - } - - private void createIndexForAttribute(AtlasGraphManagement management, String typeName, AttributeInfo field) { - final String propertyName = GraphHelper.encodePropertyKey(typeName + "." + field.name); - switch (field.dataType().getTypeCategory()) { - case PRIMITIVE: - AtlasCardinality cardinality = getCardinality(field.multiplicity); - createIndexes(management, propertyName, getPrimitiveClass(field.dataType()), field.isUnique, - cardinality, false, field.isIndexable); - break; - - case ENUM: - cardinality = getCardinality(field.multiplicity); - createIndexes(management, propertyName, String.class, field.isUnique, cardinality, false, field.isIndexable); - break; - - case ARRAY: - createLabelIfNeeded(management, propertyName, field.dataType().getName()); - break; - case MAP: - // todo - how do we overcome this limitation? - // IGNORE: Can only index single-valued property keys on vertices in Mixed Index - break; - - case STRUCT: - StructType structType = (StructType) field.dataType(); - createIndexForFields(management, structType, structType.fieldMapping().fields); - break; - - case TRAIT: - // do nothing since this is NOT contained in other types - break; - - case CLASS: - createEdgeLabel(management, propertyName); - break; - - default: - throw new IllegalArgumentException("bad data type" + field.dataType().getName()); - } - } - private void createEdgeLabel(final AtlasGraphManagement management, final String propertyName) { // Create the edge label upfront to avoid running into concurrent call issue (ATLAS-2092) // ATLAS-2092 addresses this problem by creating the edge label upfront while type creation @@ -506,50 +448,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang } } - private Class getPrimitiveClass(IDataType dataType) { - if (dataType == DataTypes.STRING_TYPE) { - return String.class; - } else if (dataType == DataTypes.SHORT_TYPE) { - return Short.class; - } else if (dataType == DataTypes.INT_TYPE) { - return Integer.class; - } else if (dataType == DataTypes.BIGINTEGER_TYPE) { - return BigInteger.class; - } else if (dataType == DataTypes.BOOLEAN_TYPE) { - return Boolean.class; - } else if (dataType == DataTypes.BYTE_TYPE) { - return Byte.class; - } else if (dataType == DataTypes.LONG_TYPE) { - return Long.class; - } else if (dataType == DataTypes.FLOAT_TYPE) { - return Float.class; - } else if (dataType == DataTypes.DOUBLE_TYPE) { - return Double.class; - } else if (dataType == DataTypes.BIGDECIMAL_TYPE) { - return BigDecimal.class; - } else if (dataType == DataTypes.DATE_TYPE) { - //Indexing with date converted to long as of now since Titan is yet to add support for Date type with mixed indexes - return Long.class; - } - - - throw new IllegalArgumentException("unknown data type " + dataType); - } - - - private AtlasCardinality getCardinality(Multiplicity multiplicity) { - if (multiplicity == Multiplicity.OPTIONAL || multiplicity == Multiplicity.REQUIRED) { - return AtlasCardinality.SINGLE; - } else if (multiplicity == Multiplicity.COLLECTION) { - return AtlasCardinality.LIST; - } else if (multiplicity == Multiplicity.SET) { - return AtlasCardinality.SET; - } - - // todo - default to LIST as this is the most forgiving - return AtlasCardinality.LIST; - } - private AtlasPropertyKey createIndexes(AtlasGraphManagement management, String propertyName, Class propertyClass, boolean isUnique, AtlasCardinality cardinality, boolean createCompositeForAttribute, boolean createCompositeWithTypeandSuperTypes) { @@ -677,70 +575,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang } } - /** - * Initialize global indices for Titan graph on server activation. - * - * Since the indices are shared state, we need to do this only from an active instance. - */ - @Override - public void instanceIsActive() throws AtlasException { - LOG.info("Reacting to active: initializing index"); - try { - initialize(); - } catch (RepositoryException | IndexException e) { - throw new AtlasException("Error in reacting to active on initialization", e); - } - } - - @Override - public void instanceIsPassive() { - LOG.info("Reacting to passive state: No action right now."); - } - - @Override - public int getHandlerOrder() { - return HandlerOrder.GRAPH_BACKED_SEARCH_INDEXER.getOrder(); - } - - @Override - public void onChange(ChangedTypeDefs changedTypeDefs) throws AtlasBaseException { - if (LOG.isDebugEnabled()) { - LOG.debug("Processing changed typedefs {}", changedTypeDefs); - } - AtlasGraphManagement management = null; - try { - management = provider.get().getManagementSystem(); - - // Update index for newly created types - if (CollectionUtils.isNotEmpty(changedTypeDefs.getCreateTypeDefs())) { - for (AtlasBaseTypeDef typeDef : changedTypeDefs.getCreateTypeDefs()) { - updateIndexForTypeDef(management, typeDef); - } - } - - // Update index for updated types - if (CollectionUtils.isNotEmpty(changedTypeDefs.getUpdatedTypeDefs())) { - for (AtlasBaseTypeDef typeDef : changedTypeDefs.getUpdatedTypeDefs()) { - updateIndexForTypeDef(management, typeDef); - } - } - - // Invalidate the property key for deleted types - if (CollectionUtils.isNotEmpty(changedTypeDefs.getDeletedTypeDefs())) { - for (AtlasBaseTypeDef typeDef : changedTypeDefs.getDeletedTypeDefs()) { - cleanupIndices(management, typeDef); - } - } - - //Commit indexes - commit(management); - } catch (RepositoryException | IndexException e) { - LOG.error("Failed to update indexes for changed typedefs", e); - attemptRollback(changedTypeDefs, management); - } - - } - private void cleanupIndices(AtlasGraphManagement management, AtlasBaseTypeDef typeDef) { Preconditions.checkNotNull(typeDef, "Cannot process null typedef"); if (LOG.isDebugEnabled()) { @@ -816,14 +650,4 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang addIndexForType(management, typeDef); LOG.info("Index creation for type {} complete", typeDef.getName()); } - - /* Commenting this out since we do not need an index for edge label here - private void createEdgeMixedIndex(String propertyName) { - EdgeLabel edgeLabel = management.getEdgeLabel(propertyName); - if (edgeLabel == null) { - edgeLabel = management.makeEdgeLabel(propertyName).make(); - management.buildEdgeIndex(edgeLabel, propertyName, Direction.BOTH, Order.DEFAULT); - LOG.info("Created index for edge label {}", propertyName); - } - }*/ }
