ATLAS-1467: instance create/full-Update implementation Signed-off-by: Madhan Neethiraj <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/2f1cb57a Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/2f1cb57a Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/2f1cb57a Branch: refs/heads/master Commit: 2f1cb57a757e15c97a2b1437e282be48c553d4e8 Parents: 511c886 Author: Suma Shivaprasad <[email protected]> Authored: Thu Jan 19 17:47:00 2017 -0800 Committer: Madhan Neethiraj <[email protected]> Committed: Thu Jan 19 17:51:04 2017 -0800 ---------------------------------------------------------------------- .../java/org/apache/atlas/AtlasErrorCode.java | 7 +- .../atlas/model/instance/AtlasEntity.java | 14 +- .../atlas/model/instance/AtlasEntityHeader.java | 11 +- .../atlas/model/instance/EntityMutations.java | 3 +- .../atlas/model/typedef/AtlasStructDef.java | 5 +- .../atlas/type/AtlasClassificationType.java | 63 +-- .../org/apache/atlas/type/AtlasEntityType.java | 114 ++-- .../org/apache/atlas/type/AtlasStructType.java | 139 ++++- .../test/java/org/apache/atlas/TestUtilsV2.java | 74 ++- .../atlas/type/TestAtlasTypeRegistry.java | 10 +- release-log.txt | 4 + .../apache/atlas/RepositoryMetadataModule.java | 26 +- .../atlas/discovery/EntityDiscoveryService.java | 2 +- .../atlas/discovery/EntityLineageService.java | 2 +- .../atlas/repository/graph/GraphHelper.java | 2 +- .../store/graph/AtlasEntityStore.java | 8 +- .../store/graph/EntityGraphDiscovery.java | 39 ++ .../graph/EntityGraphDiscoveryContext.java | 170 ++++++ .../repository/store/graph/EntityResolver.java | 30 ++ .../store/graph/v1/ArrayVertexMapper.java | 179 ++++++ .../graph/v1/AtlasEntityGraphDiscoveryV1.java | 244 +++++++++ .../store/graph/v1/AtlasEntityStoreV1.java | 139 ++++- .../store/graph/v1/AtlasEnumDefStoreV1.java | 12 +- .../store/graph/v1/AtlasGraphUtilsV1.java | 82 ++- .../store/graph/v1/AtlasStructDefStoreV1.java | 18 +- .../store/graph/v1/DeleteHandlerV1.java | 539 +++++++++++++++++++ .../store/graph/v1/EntityGraphMapper.java | 185 +++++++ .../store/graph/v1/EntityMutationContext.java | 124 +++++ .../store/graph/v1/GraphMutationContext.java | 195 +++++++ .../store/graph/v1/HardDeleteHandlerV1.java | 43 ++ .../store/graph/v1/IDBasedEntityResolver.java | 118 ++++ .../store/graph/v1/InstanceGraphMapper.java | 39 ++ .../store/graph/v1/MapVertexMapper.java | 200 +++++++ .../store/graph/v1/SoftDeleteHandlerV1.java | 72 +++ .../store/graph/v1/StructVertexMapper.java | 203 +++++++ .../graph/v1/UniqAttrBasedEntityResolver.java | 136 +++++ .../util/AtlasRepositoryConfiguration.java | 12 + .../atlas/lineage/EntityLineageServiceTest.java | 12 +- .../store/graph/AtlasTypeDefGraphStoreTest.java | 3 +- .../store/graph/v1/AtlasEntityStoreV1Test.java | 236 ++++++++ .../java/org/apache/atlas/RequestContextV1.java | 122 +++++ .../adapters/AtlasEntityFormatConverter.java | 4 +- .../web/adapters/AtlasInstanceRestAdapters.java | 4 +- .../adapters/AtlasStructFormatConverter.java | 32 +- .../org/apache/atlas/web/rest/EntitiesREST.java | 2 +- .../org/apache/atlas/web/util/LineageUtils.java | 2 +- .../atlas/web/adapters/TestEntitiesREST.java | 4 +- .../atlas/web/adapters/TestEntityREST.java | 4 +- .../atlas/web/resources/BaseResourceIT.java | 6 +- .../EntityDiscoveryJerseyResourceIT.java | 4 +- .../web/resources/EntityV2JerseyResourceIT.java | 22 +- 51 files changed, 3450 insertions(+), 270 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2f1cb57a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java index a6438ed..f0aae0c 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java +++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java @@ -62,6 +62,7 @@ public enum AtlasErrorCode { INSTANCE_LINEAGE_INVALID_PARAMS(404, "ATLAS4046E", "Invalid lineage query parameters passed {0}: {1}"), INSTANCE_LINEAGE_QUERY_FAILED(404, "ATLAS4047E", "Instance lineage query failed {0}"), DISCOVERY_QUERY_FAILED(404, "ATLAS4048E", "Discovery query failed {0}"), + INSTANCE_CRUD_INVALID_PARAMS(404, "ATLAS4049E", "Invalid instance creation/updation parameters passed : {0}"), // All data conflict errors go here @@ -72,7 +73,11 @@ public enum AtlasErrorCode { // All internal errors go here INTERNAL_ERROR(500, "ATLAS5001E", "Internal server error {0}"), INDEX_CREATION_FAILED(500, "ATLAS5002E", "Index creation failed for {0}"), - INDEX_ROLLBACK_FAILED(500, "ATLAS5003E", "Index rollback failed for {0}"); + INDEX_ROLLBACK_FAILED(500, "ATLAS5003E", "Index rollback failed for {0}"), + + INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND(400, "ATLAS40018E", "Instance {0} with unique attribute {1} does not exist"), + + UNKNOWN_ATTRIBUTE(400, "ATLAS40019E", "Attribute {0} not found for type {1}"); private String errorCode; private String errorMessage; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2f1cb57a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java index 2ad0f76..9494fe4 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java @@ -55,15 +55,15 @@ public class AtlasEntity extends AtlasStruct implements Serializable { /** * Status of the entity - can be active or deleted. Deleted entities are not removed from Atlas store. */ - public enum Status { STATUS_ACTIVE, STATUS_DELETED } + public enum Status { ACTIVE, DELETED } private String guid = null; - private Status status = Status.STATUS_ACTIVE; + private Status status = Status.ACTIVE; private String createdBy = null; private String updatedBy = null; private Date createTime = null; private Date updateTime = null; - private Long version = null; + private Long version = new Long(0); @JsonIgnore private static AtomicLong s_nextId = new AtomicLong(System.nanoTime()); @@ -89,7 +89,6 @@ public class AtlasEntity extends AtlasStruct implements Serializable { setUpdatedBy(null); setCreateTime(null); setUpdateTime(null); - setVersion(null); } public AtlasEntity(AtlasEntity other) { @@ -247,7 +246,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable { @JsonIgnore public boolean isUnassigned() { - return guid != null && guid.length() > 0 && guid.charAt(0) == '-'; + return isUnAssigned(guid); } @JsonIgnore @@ -266,6 +265,11 @@ public class AtlasEntity extends AtlasStruct implements Serializable { return true; } + @JsonIgnore + public static boolean isUnAssigned(String guid) { + return guid != null && guid.length() > 0 && guid.charAt(0) == '-'; + } + private String nextInternalId() { return "-" + Long.toString(s_nextId.getAndIncrement()); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2f1cb57a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java index e7b70aa..5797a69 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java @@ -18,6 +18,7 @@ package org.apache.atlas.model.instance; import java.io.Serializable; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -49,7 +50,7 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable { private static final long serialVersionUID = 1L; private String guid = null; - private AtlasEntity.Status status = AtlasEntity.Status.STATUS_ACTIVE; + private AtlasEntity.Status status = AtlasEntity.Status.ACTIVE; private String displayText = null; public AtlasEntityHeader() { @@ -66,11 +67,15 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable { public AtlasEntityHeader(String typeName, Map<String, Object> attributes) { super(typeName, attributes); + } + - setGuid(null); - setStatus(null); + public AtlasEntityHeader(String typeName, String guid, Map<String, Object> attributes) { + super(typeName, attributes); + setGuid(guid); } + public AtlasEntityHeader(AtlasEntityHeader other) { super(other); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2f1cb57a/intg/src/main/java/org/apache/atlas/model/instance/EntityMutations.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/instance/EntityMutations.java b/intg/src/main/java/org/apache/atlas/model/instance/EntityMutations.java index 3501c90..74e3c57 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/EntityMutations.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/EntityMutations.java @@ -43,7 +43,8 @@ public class EntityMutations implements Serializable { private List<EntityMutation> entityMutations = new ArrayList<>(); public enum EntityOperation { - CREATE_OR_UPDATE, + CREATE, + UPDATE, PARTIAL_UPDATE, DELETE, } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2f1cb57a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java index aee26ef..2c00f54 100644 --- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java +++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java @@ -19,6 +19,7 @@ package org.apache.atlas.model.typedef; import java.io.Serializable; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -183,7 +184,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { return findAttribute(attributeDefs, attrName) != null; } - private static AtlasAttributeDef findAttribute(List<AtlasAttributeDef> attributeDefs, String attrName) { + public static AtlasAttributeDef findAttribute(Collection<AtlasAttributeDef> attributeDefs, String attrName) { AtlasAttributeDef ret = null; if (CollectionUtils.isNotEmpty(attributeDefs)) { @@ -451,6 +452,8 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { } } + + /** * class that captures details of a constraint. */ http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2f1cb57a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java index 8772720..7d89848 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java @@ -46,8 +46,6 @@ public class AtlasClassificationType extends AtlasStructType { private List<AtlasClassificationType> superTypes = Collections.emptyList(); private Set<String> allSuperTypes = Collections.emptySet(); - private Map<String, AtlasAttributeDef> allAttributeDefs = Collections.emptyMap(); - private Map<String, AtlasType> allAttributeTypes = new HashMap<>(); public AtlasClassificationType(AtlasClassificationDef classificationDef) { super(classificationDef); @@ -72,7 +70,7 @@ public class AtlasClassificationType extends AtlasStructType { List<AtlasClassificationType> s = new ArrayList<>(); Set<String> allS = new HashSet<>(); - Map<String, AtlasAttributeDef> allA = new HashMap<>(); + Map<String, AtlasAttribute> allA = new HashMap<>(); getTypeHierarchyInfo(typeRegistry, allS, allA); @@ -89,8 +87,7 @@ public class AtlasClassificationType extends AtlasStructType { this.superTypes = Collections.unmodifiableList(s); this.allSuperTypes = Collections.unmodifiableSet(allS); - this.allAttributeDefs = Collections.unmodifiableMap(allA); - this.allAttributeTypes = new HashMap<>(); // this will be rebuilt on calls to getAttributeType() + this.allAttributes = Collections.unmodifiableMap(allA); } public Set<String> getSuperTypes() { @@ -99,51 +96,6 @@ public class AtlasClassificationType extends AtlasStructType { public Set<String> getAllSuperTypes() { return allSuperTypes; } - public Map<String, AtlasAttributeDef> getAllAttributeDefs() { return allAttributeDefs; } - - @Override - public AtlasType getAttributeType(String attributeName) { - AtlasType ret = allAttributeTypes.get(attributeName); - - if (ret == null) { - ret = super.getAttributeType(attributeName); - - if (ret == null) { - for (AtlasClassificationType superType : superTypes) { - ret = superType.getAttributeType(attributeName); - - if (ret != null) { - break; - } - } - } - - if (ret != null) { - allAttributeTypes.put(attributeName, ret); - } - } - - return ret; - } - - - @Override - public AtlasAttributeDef getAttributeDef(String attributeName) { - AtlasAttributeDef ret = super.getAttributeDef(attributeName); - - if (ret == null) { - for (AtlasClassificationType superType : superTypes) { - ret = superType.getAttributeDef(attributeName); - - if (ret != null) { - break; - } - } - } - - return ret; - } - public boolean isSuperTypeOf(AtlasClassificationType classificationType) { return classificationType != null && classificationType.getAllSuperTypes().contains(this.getTypeName()); } @@ -243,10 +195,10 @@ public class AtlasClassificationType extends AtlasStructType { private void getTypeHierarchyInfo(AtlasTypeRegistry typeRegistry, Set<String> allSuperTypeNames, - Map<String, AtlasAttributeDef> allAttributeDefs) throws AtlasBaseException { + Map<String, AtlasAttribute> allAttributes) throws AtlasBaseException { List<String> visitedTypes = new ArrayList<>(); - collectTypeHierarchyInfo(typeRegistry, allSuperTypeNames, allAttributeDefs, visitedTypes); + collectTypeHierarchyInfo(typeRegistry, allSuperTypeNames, allAttributes, visitedTypes); } /* @@ -255,7 +207,7 @@ public class AtlasClassificationType extends AtlasStructType { */ private void collectTypeHierarchyInfo(AtlasTypeRegistry typeRegistry, Set<String> allSuperTypeNames, - Map<String, AtlasAttributeDef> allAttributeDefs, + Map<String, AtlasAttribute> allAttributes, List<String> visitedTypes) throws AtlasBaseException { if (visitedTypes.contains(classificationDef.getName())) { throw new AtlasBaseException(AtlasErrorCode.CIRCULAR_REFERENCE, classificationDef.getName(), @@ -270,7 +222,7 @@ public class AtlasClassificationType extends AtlasStructType { if (type instanceof AtlasClassificationType) { AtlasClassificationType superType = (AtlasClassificationType) type; - superType.collectTypeHierarchyInfo(typeRegistry, allSuperTypeNames, allAttributeDefs, visitedTypes); + superType.collectTypeHierarchyInfo(typeRegistry, allSuperTypeNames, allAttributes, visitedTypes); } } visitedTypes.remove(classificationDef.getName()); @@ -280,7 +232,8 @@ public class AtlasClassificationType extends AtlasStructType { if (CollectionUtils.isNotEmpty(classificationDef.getAttributeDefs())) { for (AtlasAttributeDef attributeDef : classificationDef.getAttributeDefs()) { - allAttributeDefs.put(attributeDef.getName(), attributeDef); + AtlasType type = typeRegistry.getType(attributeDef.getTypeName()); + allAttributes.put(attributeDef.getName(), new AtlasAttribute(this, classificationDef, attributeDef, type)); } } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2f1cb57a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java index 3625f72..caadecc 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java @@ -21,9 +21,11 @@ package org.apache.atlas.type; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.typedef.AtlasEntityDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,8 +47,6 @@ public class AtlasEntityType extends AtlasStructType { private List<AtlasEntityType> superTypes = Collections.emptyList(); private Set<String> allSuperTypes = Collections.emptySet(); - private Map<String, AtlasAttributeDef> allAttributeDefs = Collections.emptyMap(); - private Map<String, AtlasType> allAttributeTypes = new HashMap<>(); public AtlasEntityType(AtlasEntityDef entityDef) { super(entityDef); @@ -70,7 +70,7 @@ public class AtlasEntityType extends AtlasStructType { List<AtlasEntityType> s = new ArrayList<>(); Set<String> allS = new HashSet<>(); - Map<String, AtlasAttributeDef> allA = new HashMap<>(); + Map<String, AtlasAttribute> allA = new HashMap<>(); getTypeHierarchyInfo(typeRegistry, allS, allA); @@ -86,8 +86,7 @@ public class AtlasEntityType extends AtlasStructType { this.superTypes = Collections.unmodifiableList(s); this.allSuperTypes = Collections.unmodifiableSet(allS); - this.allAttributeDefs = Collections.unmodifiableMap(allA); - this.allAttributeTypes = new HashMap<>(); // this will be rebuilt on calls to getAttributeType() + this.allAttributes = Collections.unmodifiableMap(allA); } public Set<String> getSuperTypes() { @@ -98,50 +97,6 @@ public class AtlasEntityType extends AtlasStructType { return allSuperTypes; } - public Map<String, AtlasAttributeDef> getAllAttributeDefs() { return allAttributeDefs; } - - @Override - public AtlasType getAttributeType(String attributeName) { - AtlasType ret = allAttributeTypes.get(attributeName); - - if (ret == null) { - ret = super.getAttributeType(attributeName); - - if (ret == null) { - for (AtlasEntityType superType : superTypes) { - ret = superType.getAttributeType(attributeName); - - if (ret != null) { - break; - } - } - } - - if (ret != null) { - allAttributeTypes.put(attributeName, ret); - } - } - - return ret; - } - - @Override - public AtlasAttributeDef getAttributeDef(String attributeName) { - AtlasAttributeDef ret = super.getAttributeDef(attributeName); - - if (ret == null) { - for (AtlasEntityType superType : superTypes) { - ret = superType.getAttributeDef(attributeName); - - if (ret != null) { - break; - } - } - } - - return ret; - } - public boolean isSuperTypeOf(AtlasEntityType entityType) { return entityType != null && entityType.getAllSuperTypes().contains(this.getTypeName()); } @@ -150,6 +105,10 @@ public class AtlasEntityType extends AtlasStructType { return entityType != null && allSuperTypes.contains(entityType.getTypeName()); } + public boolean isSubTypeOf(String entityTypeName) { + return StringUtils.isNotEmpty(entityTypeName) && allSuperTypes.contains(entityTypeName); + } + @Override public AtlasEntity createDefaultValue() { AtlasEntity ret = new AtlasEntity(entityDef.getName()); @@ -162,13 +121,17 @@ public class AtlasEntityType extends AtlasStructType { @Override public boolean isValidValue(Object obj) { if (obj != null) { - for (AtlasEntityType superType : superTypes) { - if (!superType.isValidValue(obj)) { - return false; + if (obj instanceof AtlasObjectId) { + AtlasObjectId objId = (AtlasObjectId ) obj; + return validateAtlasObjectId(objId); + } else { + for (AtlasEntityType superType : superTypes) { + if (!superType.isValidValue(obj)) { + return false; + } } + return super.isValidValue(obj); } - - return super.isValidValue(obj); } return true; @@ -186,6 +149,8 @@ public class AtlasEntityType extends AtlasStructType { } else if (obj instanceof Map) { normalizeAttributeValues((Map) obj); ret = obj; + } else if (obj instanceof AtlasObjectId) { + ret = obj; } } } @@ -194,10 +159,20 @@ public class AtlasEntityType extends AtlasStructType { } @Override + public AtlasAttribute getAttribute(String attributeName) { + return findAttribute(allAttributes.values(), attributeName); + } + + @Override public boolean validateValue(Object obj, String objName, List<String> messages) { boolean ret = true; if (obj != null) { + if (obj instanceof AtlasObjectId) { + AtlasObjectId objId = (AtlasObjectId ) obj; + return validateAtlasObjectId(objId); + } + for (AtlasEntityType superType : superTypes) { ret = superType.validateValue(obj, objName, messages) && ret; } @@ -241,10 +216,10 @@ public class AtlasEntityType extends AtlasStructType { private void getTypeHierarchyInfo(AtlasTypeRegistry typeRegistry, Set<String> allSuperTypeNames, - Map<String, AtlasAttributeDef> allAttributeDefs) throws AtlasBaseException { + Map<String, AtlasAttribute> allAttributes) throws AtlasBaseException { List<String> visitedTypes = new ArrayList<>(); - collectTypeHierarchyInfo(typeRegistry, allSuperTypeNames, allAttributeDefs, visitedTypes); + collectTypeHierarchyInfo(typeRegistry, allSuperTypeNames, allAttributes, visitedTypes); } /* @@ -253,7 +228,7 @@ public class AtlasEntityType extends AtlasStructType { */ private void collectTypeHierarchyInfo(AtlasTypeRegistry typeRegistry, Set<String> allSuperTypeNames, - Map<String, AtlasAttributeDef> allAttributeDefs, + Map<String, AtlasAttribute> allAttributes, List<String> visitedTypes) throws AtlasBaseException { if (visitedTypes.contains(entityDef.getName())) { throw new AtlasBaseException(AtlasErrorCode.CIRCULAR_REFERENCE, entityDef.getName(), @@ -267,19 +242,36 @@ public class AtlasEntityType extends AtlasStructType { if (type instanceof AtlasEntityType) { AtlasEntityType superType = (AtlasEntityType) type; - - superType.collectTypeHierarchyInfo(typeRegistry, allSuperTypeNames, allAttributeDefs, visitedTypes); + superType.collectTypeHierarchyInfo(typeRegistry, allSuperTypeNames, allAttributes, visitedTypes); } } visitedTypes.remove(entityDef.getName()); - allSuperTypeNames.addAll(entityDef.getSuperTypes()); } if (CollectionUtils.isNotEmpty(entityDef.getAttributeDefs())) { for (AtlasAttributeDef attributeDef : entityDef.getAttributeDefs()) { - allAttributeDefs.put(attributeDef.getName(), attributeDef); + + AtlasType type = typeRegistry.getType(attributeDef.getTypeName()); + allAttributes.put(attributeDef.getName(), new AtlasAttribute(this, entityDef, attributeDef, type)); + } + } + } + + private boolean validateAtlasObjectId(AtlasObjectId objId) { + if (StringUtils.isEmpty(objId.getTypeName()) || StringUtils.isEmpty(objId.getGuid())) { + return false; + } else { + String typeName = objId.getTypeName(); + if (!typeName.equals(getTypeName())) { + //TODO - Enable below after enabling subType check +// if ( !isSuperTypeOf(typeName)) { +// return false; +// } + return false; } } + return AtlasEntity.isAssigned(objId.getGuid()) || AtlasEntity.isUnAssigned((objId.getGuid())); } + } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2f1cb57a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java index e20af76..4712508 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java @@ -20,6 +20,8 @@ package org.apache.atlas.type; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.TypeCategory; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; @@ -30,6 +32,7 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -49,10 +52,9 @@ public class AtlasStructType extends AtlasType { private final AtlasStructDef structDef; - private Map<String, AtlasType> attrTypes = Collections.emptyMap(); private Set<String> foreignKeyAttributes = new HashSet<>(); private Map<String, TypeAttributePair> mappedFromRefAttributes = new HashMap<>(); - + protected Map<String, AtlasAttribute> allAttributes = Collections.emptyMap(); public AtlasStructType(AtlasStructDef structDef) { super(structDef); @@ -70,9 +72,12 @@ public class AtlasStructType extends AtlasType { public AtlasStructDef getStructDef() { return structDef; } - public AtlasType getAttributeType(String attributeName) { return attrTypes.get(attributeName); } + public AtlasType getAttributeType(String attributeName) { + AtlasAttribute attribute = allAttributes.get(attributeName); + return attribute != null ? attribute.getAttributeType() : null; + } - public AtlasAttributeDef getAttributeDef(String attributeName) { return structDef.getAttribute(attributeName); } + public AtlasAttributeDef getAttributeDef(String attributeName) { return allAttributes.get(attributeName) != null ? allAttributes.get(attributeName).getAttributeDef() : null; } public boolean isForeignKeyAttribute(String attributeName) { return foreignKeyAttributes.contains(attributeName); @@ -101,10 +106,12 @@ public class AtlasStructType extends AtlasType { @Override public void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { - Map<String, AtlasType> a = new HashMap<>(); + Map<String, AtlasAttribute> a = new HashMap<>(); for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) { + AtlasType attrType = typeRegistry.getType(attributeDef.getTypeName()); + AtlasAttribute attribute = new AtlasAttribute(this, structDef, attributeDef, attrType); resolveConstraints(attributeDef, attrType); @@ -122,10 +129,10 @@ public class AtlasStructType extends AtlasType { arrayType.setMaxCount(attributeDef.getValuesMaxCount()); } - a.put(attributeDef.getName(), attrType); + a.put(attributeDef.getName(), attribute); } - this.attrTypes = Collections.unmodifiableMap(a); + this.allAttributes = Collections.unmodifiableMap(a); } @Override @@ -137,6 +144,29 @@ public class AtlasStructType extends AtlasType { return ret; } + public Map<String, AtlasAttribute> getAllAttributes() { + return allAttributes; + } + + public AtlasAttribute getAttribute(String attributeName) { + return findAttribute(allAttributes.values(), attributeName); + } + + public static AtlasAttribute findAttribute(Collection<AtlasAttribute> attributes, String attrName) { + AtlasAttribute ret = null; + + if (CollectionUtils.isNotEmpty(attributes)) { + for (AtlasAttribute attribute : attributes) { + if (org.apache.hadoop.util.StringUtils.equalsIgnoreCase(attribute.getAttributeDef().getName(), attrName)) { + ret = attribute; + break; + } + } + } + + return ret; + } + @Override public boolean isValidValue(Object obj) { if (obj != null) { @@ -157,7 +187,7 @@ public class AtlasStructType extends AtlasType { } } } else { - return false; // invalid type + return false; } } @@ -193,9 +223,11 @@ public class AtlasStructType extends AtlasType { for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) { String attrName = attributeDef.getName(); - AtlasType dataType = attrTypes.get(attributeDef.getName()); - if (dataType != null) { + AtlasAttribute attribute = allAttributes.get(attributeDef.getName()); + + if (attribute != null) { + AtlasType dataType = attribute.getAttributeType(); Object value = structObj.getAttribute(attrName); String fieldName = objName + "." + attrName; @@ -213,9 +245,10 @@ public class AtlasStructType extends AtlasType { for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) { String attrName = attributeDef.getName(); - AtlasType dataType = attrTypes.get(attributeDef.getName()); + AtlasAttribute attribute = allAttributes.get(attributeDef.getName()); - if (dataType != null) { + if (attribute != null) { + AtlasType dataType = attribute.getAttributeType(); Object value = map.get(attrName); String fieldName = objName + "." + attrName; @@ -230,7 +263,6 @@ public class AtlasStructType extends AtlasType { } } else { ret = false; - messages.add(objName + "=" + obj + ": invalid value for type " + getTypeName()); } } @@ -292,9 +324,10 @@ public class AtlasStructType extends AtlasType { Object ret = null; if (attributeDef != null) { - AtlasType dataType = attrTypes.get(attributeDef.getName()); + AtlasAttribute attribute = allAttributes.get(attributeDef.getName()); - if (dataType != null) { + if (attribute != null) { + AtlasType dataType = attribute.getAttributeType(); ret = dataType.createDefaultValue(); } } @@ -306,12 +339,14 @@ public class AtlasStructType extends AtlasType { boolean ret = true; if (value != null) { - AtlasType attrType = attrTypes.get(attributeDef.getName()); + AtlasAttribute attribute = allAttributes.get(attributeDef.getName()); - if (attrType != null) { - if (!attrType.isValidValue(value)) { - ret = false; // invalid value - } + if (attribute != null) { + AtlasType attrType = attribute.getAttributeType(); + + if (!attrType.isValidValue(value)) { + ret = false; // invalid value + } } } else if (!attributeDef.getIsOptional()) { ret = false; // mandatory attribute not present @@ -321,9 +356,11 @@ public class AtlasStructType extends AtlasType { } private Object getNormalizedValue(Object value, AtlasAttributeDef attributeDef) { - AtlasType attrType = attrTypes.get(attributeDef.getName()); + AtlasAttribute attribute = allAttributes.get(attributeDef.getName()); + + if (attribute != null) { + AtlasType attrType = attribute.getAttributeType(); - if (attrType != null) { if (value == null) { if (!attributeDef.getIsOptional()) { return attrType.createDefaultValue(); @@ -419,8 +456,8 @@ public class AtlasStructType extends AtlasType { String.valueOf(constraintDef.getParams())); } - AtlasStructType structType = (AtlasStructType)attribType; - AtlasAttributeDef refAttrib = structType.getAttributeDef(refAttribName); + AtlasStructType structType = (AtlasStructType) attribType; + AtlasAttributeDef refAttrib = structType.getStructDef().getAttribute(refAttribName); if (refAttrib == null) { throw new AtlasBaseException(AtlasErrorCode.CONSTRAINT_NOT_EXIST, @@ -447,4 +484,58 @@ public class AtlasStructType extends AtlasType { this.attributeName = attributeName; } } + + public String getQualifiedAttributeName(String attrName) throws AtlasBaseException { + if ( allAttributes.containsKey(attrName)) { + return allAttributes.get(attrName).getQualifiedName(); + } + + throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attrName, structDef.getName()); + } + + public static class AtlasAttribute { + + private final AtlasStructType structType; + private final AtlasStructDef structDef; + private final AtlasType attributeType; + private final AtlasAttributeDef attributeDef; + private final String qualifiedName; + + public AtlasAttribute(AtlasStructType structType, AtlasStructDef structDef, AtlasAttributeDef attrDef, AtlasType attributeType) { + this.structType = structType; + this.structDef = structDef; + this.attributeDef = attrDef; + this.attributeType = attributeType; + this.qualifiedName = getQualifiedAttributeName(structDef, attributeDef.getName()); + } + + public AtlasStructType getStructType() { + return structType; + } + + public String getQualifiedName() { + return qualifiedName; + } + + public AtlasStructDef getStructDef() { + return structDef; + } + + public AtlasType getAttributeType() { + return attributeType; + } + + public AtlasAttributeDef getAttributeDef() { + return attributeDef; + } + + public String getQualifiedAttributeName() { + return qualifiedName; + } + + public static String getQualifiedAttributeName(AtlasStructDef structDef, String attrName) { + final String typeName = structDef.getName(); + return attrName.contains(".") ? attrName : String.format("%s.%s", typeName, attrName); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2f1cb57a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java index 53b109c..f9040f3 100755 --- a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java +++ b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java @@ -21,6 +21,7 @@ package org.apache.atlas; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.model.typedef.AtlasClassificationDef; import org.apache.atlas.model.typedef.AtlasEntityDef; @@ -80,13 +81,19 @@ public final class TestUtilsV2 { AtlasEntityDef deptTypeDef = AtlasTypeUtil.createClassTypeDef(DEPARTMENT_TYPE, "Department"+_description, ImmutableSet.<String>of(), - AtlasTypeUtil.createRequiredAttrDef("name", "string"), - new AtlasAttributeDef("employees", String.format("array<%s>", "Person"), true, + AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), + new AtlasAttributeDef("employees", String.format("array<%s>", "Employee"), true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, - Collections.<AtlasStructDef.AtlasConstraintDef>emptyList())); + new ArrayList<AtlasStructDef.AtlasConstraintDef>())); + + deptTypeDef.getAttribute("employees").addConstraint( + new AtlasStructDef.AtlasConstraintDef( + AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_MAPPED_FROM_REF, new HashMap<String, Object>() {{ + put(AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_REF_ATTRIBUTE, "department"); + }})); AtlasEntityDef personTypeDef = AtlasTypeUtil.createClassTypeDef("Person", "Person"+_description, ImmutableSet.<String>of(), - AtlasTypeUtil.createRequiredAttrDef("name", "string"), + AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), AtlasTypeUtil.createOptionalAttrDef("address", "Address"), AtlasTypeUtil.createOptionalAttrDef("birthday", "date"), AtlasTypeUtil.createOptionalAttrDef("hasPets", "boolean"), @@ -103,20 +110,25 @@ public final class TestUtilsV2 { new AtlasAttributeDef("department", "Department", false, AtlasAttributeDef.Cardinality.SINGLE, 1, 1, false, false, - Collections.<AtlasStructDef.AtlasConstraintDef>emptyList()), - new AtlasAttributeDef("manager", "Employee", true, + new ArrayList<AtlasStructDef.AtlasConstraintDef>()), + new AtlasAttributeDef("manager", "Manager", true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasStructDef.AtlasConstraintDef>emptyList()), - new AtlasAttributeDef("mentor", "Employee", true, + new AtlasAttributeDef("mentor", EMPLOYEE_TYPE, true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasStructDef.AtlasConstraintDef>emptyList()), AtlasTypeUtil.createOptionalAttrDef("shares", "long"), AtlasTypeUtil.createOptionalAttrDef("salary", "double") - ); + employeeTypeDef.getAttribute("department").addConstraint( + new AtlasStructDef.AtlasConstraintDef( + AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_FOREIGN_KEY, new HashMap<String, Object>() {{ + put(AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_ON_DELETE, AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_VAL_CASCADE); + }})); + AtlasEntityDef managerTypeDef = AtlasTypeUtil.createClassTypeDef("Manager", "Manager"+_description, ImmutableSet.of("Employee"), new AtlasAttributeDef("subordinates", String.format("array<%s>", "Employee"), false, AtlasAttributeDef.Cardinality.SET, 1, 10, false, false, @@ -149,7 +161,7 @@ public final class TestUtilsV2 { AtlasEntityDef deptTypeDef = AtlasTypeUtil.createClassTypeDef(DEPARTMENT_TYPE, "Department"+_description, ImmutableSet.<String>of(), - AtlasTypeUtil.createRequiredAttrDef("name", "string"), + AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), AtlasTypeUtil.createOptionalAttrDef("dep-code", "string"), new AtlasAttributeDef("employees", String.format("array<%s>", "Employee"), true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, @@ -157,7 +169,7 @@ public final class TestUtilsV2 { AtlasEntityDef personTypeDef = AtlasTypeUtil.createClassTypeDef("Person", "Person"+_description, ImmutableSet.<String>of(), - AtlasTypeUtil.createRequiredAttrDef("name", "string"), + AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), AtlasTypeUtil.createOptionalAttrDef("email", "string"), AtlasTypeUtil.createOptionalAttrDef("address", "Address"), AtlasTypeUtil.createOptionalAttrDef("birthday", "date"), @@ -178,11 +190,11 @@ public final class TestUtilsV2 { AtlasAttributeDef.Cardinality.SINGLE, 1, 1, false, false, Collections.<AtlasStructDef.AtlasConstraintDef>emptyList()), - new AtlasAttributeDef("manager", "Employee", true, + new AtlasAttributeDef("manager", "Manager", true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasStructDef.AtlasConstraintDef>emptyList()), - new AtlasAttributeDef("mentor", "Employee", true, + new AtlasAttributeDef("mentor", EMPLOYEE_TYPE, true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasStructDef.AtlasConstraintDef>emptyList()), @@ -241,7 +253,7 @@ public final class TestUtilsV2 { AtlasAttributeDef.Cardinality.SINGLE, 1, 1, false, false, Collections.<AtlasStructDef.AtlasConstraintDef>emptyList()), - new AtlasAttributeDef("manager", "Person", true, + new AtlasAttributeDef("manager", "Manager", true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasStructDef.AtlasConstraintDef>emptyList()), @@ -269,24 +281,24 @@ public final class TestUtilsV2 { public static final String DEPARTMENT_TYPE = "Department"; public static final String PERSON_TYPE = "Person"; + public static final String EMPLOYEE_TYPE = "Employee"; public static AtlasEntity createDeptEg1() { AtlasEntity hrDept = new AtlasEntity(DEPARTMENT_TYPE); - AtlasEntity john = new AtlasEntity(PERSON_TYPE); + AtlasEntity john = new AtlasEntity(EMPLOYEE_TYPE); -// AtlasEntity jane = new AtlasEntity("Manager", "SecurityClearance"); AtlasEntity jane = new AtlasEntity("Manager"); AtlasEntity johnAddr = new AtlasEntity("Address"); AtlasEntity janeAddr = new AtlasEntity("Address"); AtlasEntity julius = new AtlasEntity("Manager"); AtlasEntity juliusAddr = new AtlasEntity("Address"); - AtlasEntity max = new AtlasEntity("Person"); + AtlasEntity max = new AtlasEntity(EMPLOYEE_TYPE); AtlasEntity maxAddr = new AtlasEntity("Address"); - + AtlasObjectId deptId = new AtlasObjectId(hrDept.getTypeName(), hrDept.getGuid()); hrDept.setAttribute("name", "hr"); john.setAttribute("name", "John"); - john.setAttribute("department", hrDept); + john.setAttribute("department", deptId); johnAddr.setAttribute("street", "Stewart Drive"); johnAddr.setAttribute("city", "Sunnyvale"); john.setAttribute("address", johnAddr); @@ -303,26 +315,32 @@ public final class TestUtilsV2 { john.setAttribute("approximationOfPi", new BigDecimal("3.141592653589793238462643383279502884197169399375105820974944592307816406286")); jane.setAttribute("name", "Jane"); - jane.setAttribute("department", hrDept); + jane.setAttribute("department", deptId); janeAddr.setAttribute("street", "Great America Parkway"); janeAddr.setAttribute("city", "Santa Clara"); jane.setAttribute("address", janeAddr); janeAddr.setAttribute("street", "Great America Parkway"); julius.setAttribute("name", "Julius"); - julius.setAttribute("department", hrDept); + julius.setAttribute("department", deptId); juliusAddr.setAttribute("street", "Madison Ave"); juliusAddr.setAttribute("city", "Newtonville"); julius.setAttribute("address", juliusAddr); julius.setAttribute("subordinates", ImmutableList.of()); + AtlasObjectId janeId = new AtlasObjectId(jane.getTypeName(), jane.getGuid()); + + //TODO - Change to MANAGER_TYPE for JULIUS + AtlasObjectId maxId = new AtlasObjectId(EMPLOYEE_TYPE, max.getGuid()); + AtlasObjectId juliusId = new AtlasObjectId(EMPLOYEE_TYPE, julius.getGuid()); + max.setAttribute("name", "Max"); - max.setAttribute("department", hrDept); + max.setAttribute("department", deptId); maxAddr.setAttribute("street", "Ripley St"); maxAddr.setAttribute("city", "Newton"); max.setAttribute("address", maxAddr); - max.setAttribute("manager", jane); - max.setAttribute("mentor", julius); + max.setAttribute("manager", janeId); + max.setAttribute("mentor", juliusId); max.setAttribute("birthday",new Date(1979, 3, 15)); max.setAttribute("hasPets", true); max.setAttribute("age", 36); @@ -334,15 +352,15 @@ public final class TestUtilsV2 { max.setAttribute("numberOfStarsEstimate", new BigInteger("1000000000000000000000000000000")); max.setAttribute("approximationOfPi", new BigDecimal("3.1415926535897932")); - john.setAttribute("manager", jane); - john.setAttribute("mentor", max); + john.setAttribute("manager", janeId); + john.setAttribute("mentor", maxId); hrDept.setAttribute("employees", ImmutableList.of(john, jane, julius, max)); jane.setAttribute("subordinates", ImmutableList.of(john, max)); - Map<String, Integer> secClearanceLevelMap = new HashMap<>(); - secClearanceLevelMap.put("level", 1); - jane.setAttribute("SecurityClearance", secClearanceLevelMap); +// Map<String, Integer> secClearanceLevelMap = new HashMap<>(); +// secClearanceLevelMap.put("level", 1); +// jane.setAttribute("SecurityClearance", secClearanceLevelMap); return hrDept; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2f1cb57a/intg/src/test/java/org/apache/atlas/type/TestAtlasTypeRegistry.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/type/TestAtlasTypeRegistry.java b/intg/src/test/java/org/apache/atlas/type/TestAtlasTypeRegistry.java index 9429c07..d171dcf 100644 --- a/intg/src/test/java/org/apache/atlas/type/TestAtlasTypeRegistry.java +++ b/intg/src/test/java/org/apache/atlas/type/TestAtlasTypeRegistry.java @@ -355,17 +355,17 @@ public class TestAtlasTypeRegistry { } catch (AtlasBaseException excp) { } - Map<String, AtlasAttributeDef> attributeDefs = null; + Map<String, AtlasStructType.AtlasAttribute> attributes = null; if (type != null) { if (type instanceof AtlasEntityType) { - attributeDefs = ((AtlasEntityType) type).getAllAttributeDefs(); + attributes = ((AtlasEntityType) type).getAllAttributes(); } else if (type instanceof AtlasClassificationType) { - attributeDefs = ((AtlasClassificationType) type).getAllAttributeDefs(); + attributes = ((AtlasClassificationType) type).getAllAttributes(); } } - assertNotNull(attributeDefs); - assertEquals(attributeDefs.keySet(), attributeNames); + assertNotNull(attributes); + assertEquals(attributes.keySet(), attributeNames); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2f1cb57a/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 92594cf..e9587c3 100644 --- a/release-log.txt +++ b/release-log.txt @@ -9,7 +9,11 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai) ALL CHANGES: +ATLAS-1467 instance create/full-Update implementation (sumasai via mneethiraj) +ATLAS-1463 option to exclude specific entity attributes in audit records ([email protected] via mneethiraj) ATLAS-1386 Avoid uunnecessary type cache lookups (jnhagelb) +ATLAS-1000 added build instructions to README.txt (mneethiraj) +ATLAS-1471 avoid unnecessary overhead in debug log calls (mneethiraj) ATLAS-1464 option to include only specified attributes in notification message ([email protected] via mneethiraj) ATLAS-1460 v2 search API updated to return name/description/owner and classification names in result (vimalsharma via mneethiraj) ATLAS-1434 fixed unit test to use correct type names; updated error message per review comments (ashutoshm via mneethiraj) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2f1cb57a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java index 54dda50..c4d5020 100755 --- a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java +++ b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java @@ -24,6 +24,7 @@ import com.google.inject.matcher.Matchers; import com.google.inject.multibindings.Multibinder; import org.aopalliance.intercept.MethodInterceptor; +import org.apache.atlas.discovery.AtlasDiscoveryService; import org.apache.atlas.discovery.AtlasLineageService; import org.apache.atlas.discovery.DataSetLineageService; import org.apache.atlas.discovery.DiscoveryService; @@ -34,7 +35,6 @@ import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; import org.apache.atlas.listener.EntityChangeListener; import org.apache.atlas.listener.TypeDefChangeListener; import org.apache.atlas.listener.TypesChangeListener; -import org.apache.atlas.discovery.AtlasDiscoveryService; import org.apache.atlas.repository.MetadataRepository; import org.apache.atlas.repository.audit.EntityAuditListener; import org.apache.atlas.repository.audit.EntityAuditRepository; @@ -42,8 +42,17 @@ import org.apache.atlas.repository.graph.DeleteHandler; import org.apache.atlas.repository.graph.GraphBackedMetadataRepository; import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.repository.store.graph.EntityGraphDiscovery; +import org.apache.atlas.repository.store.graph.EntityResolver; +import org.apache.atlas.repository.store.graph.v1.ArrayVertexMapper; +import org.apache.atlas.repository.store.graph.v1.AtlasEntityGraphDiscoveryV1; import org.apache.atlas.repository.store.graph.v1.AtlasEntityStoreV1; import org.apache.atlas.repository.store.graph.v1.AtlasTypeDefGraphStoreV1; +import org.apache.atlas.repository.store.graph.v1.DeleteHandlerV1; +import org.apache.atlas.repository.store.graph.v1.EntityGraphMapper; +import org.apache.atlas.repository.store.graph.v1.IDBasedEntityResolver; +import org.apache.atlas.repository.store.graph.v1.MapVertexMapper; +import org.apache.atlas.repository.store.graph.v1.UniqAttrBasedEntityResolver; import org.apache.atlas.repository.typestore.GraphBackedTypeStore; import org.apache.atlas.repository.typestore.ITypeStore; import org.apache.atlas.service.Service; @@ -106,8 +115,21 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule { bind(DeleteHandler.class).to(AtlasRepositoryConfiguration.getDeleteHandlerImpl()).asEagerSingleton(); + bind(DeleteHandlerV1.class).to(AtlasRepositoryConfiguration.getDeleteHandlerV1Impl()).asEagerSingleton(); + bind(TypeCache.class).to(AtlasRepositoryConfiguration.getTypeCache()).asEagerSingleton(); + bind(EntityGraphMapper.class); + + bind(MapVertexMapper.class).asEagerSingleton(); + + bind(ArrayVertexMapper.class).asEagerSingleton(); + + Multibinder<EntityResolver> entityRefResolver = + Multibinder.newSetBinder(binder(), EntityResolver.class); + entityRefResolver.addBinding().to(IDBasedEntityResolver.class); + entityRefResolver.addBinding().to(UniqAttrBasedEntityResolver.class); + //Add EntityAuditListener as EntityChangeListener Multibinder<EntityChangeListener> entityChangeListenerBinder = Multibinder.newSetBinder(binder(), EntityChangeListener.class); @@ -116,6 +138,8 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule { MethodInterceptor interceptor = new GraphTransactionInterceptor(); requestInjection(interceptor); bindInterceptor(Matchers.any(), Matchers.annotatedWith(GraphTransaction.class), interceptor); + + bind(EntityGraphDiscovery.class).to(AtlasEntityGraphDiscoveryV1.class); } protected Configuration getConfiguration() { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2f1cb57a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 2be9a2d..2b4561d 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -228,7 +228,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { String state = vertex.getProperty(Constants.STATE_PROPERTY_KEY, String.class); if (state != null) { - Status status = (state.equalsIgnoreCase("ACTIVE") ? Status.STATUS_ACTIVE : Status.STATUS_DELETED); + Status status = (state.equalsIgnoreCase("ACTIVE") ? Status.ACTIVE : Status.DELETED); ret.setStatus(status); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2f1cb57a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 45e2dd2..6b7d7d3 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -184,7 +184,7 @@ public class EntityLineageService implements AtlasLineageService { ret.setDisplayText(vertex.getProperty(Constants.QUALIFIED_NAME, String.class)); String state = vertex.getProperty(Constants.STATE_PROPERTY_KEY, String.class); - Status status = (state.equalsIgnoreCase("ACTIVE") ? Status.STATUS_ACTIVE : Status.STATUS_DELETED); + Status status = (state.equalsIgnoreCase("ACTIVE") ? Status.ACTIVE : Status.DELETED); ret.setStatus(status); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2f1cb57a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index 5259249..889236c 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -967,7 +967,7 @@ public final class GraphHelper { instanceVertex.setListProperty(actualPropertyName, value); } - public static List<String> getListProperty(AtlasVertex instanceVertex, String propertyName) throws AtlasException { + public static List<String> getListProperty(AtlasVertex instanceVertex, String propertyName) { String actualPropertyName = GraphHelper.encodePropertyKey(propertyName); return instanceVertex.getListProperty(actualPropertyName); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2f1cb57a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java index f17b816..c42f95f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java @@ -25,6 +25,7 @@ import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityWithAssociations; import org.apache.atlas.model.instance.EntityMutations; import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.type.AtlasTypeRegistry; import java.util.List; @@ -36,14 +37,14 @@ public interface AtlasEntityStore { /** * Initialization */ - void init() throws AtlasBaseException; + void init(AtlasTypeRegistry typeRegistry, EntityGraphDiscovery graphDiscovery) throws AtlasBaseException; /** * Create or update an entity if it already exists. * @param entity * @return */ - EntityMutationResponse createOrUpdate(AtlasEntity entity); + EntityMutationResponse createOrUpdate(AtlasEntity entity) throws AtlasBaseException; /** @@ -175,4 +176,5 @@ public interface AtlasEntityStore { * @throws AtlasBaseException */ AtlasEntity.AtlasEntities searchEntities(SearchFilter searchFilter) throws AtlasBaseException; -} \ No newline at end of file + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2f1cb57a/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscovery.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscovery.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscovery.java new file mode 100644 index 0000000..38fca03 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscovery.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.store.graph; + + +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; + +import java.util.List; + +public interface EntityGraphDiscovery { + + + void init() throws AtlasBaseException; + + /* + * Return list of resolved and unresolved references. + * Resolved references already exist in the ATLAS repository and have an assigned unique GUID + * Unresolved attribute references result in an error if they are not composite (managed by a parent entity) + */ + EntityGraphDiscoveryContext discoverEntities(List<AtlasEntity> entities) throws AtlasBaseException; + + void cleanUp() throws AtlasBaseException; +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2f1cb57a/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscoveryContext.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscoveryContext.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscoveryContext.java new file mode 100644 index 0000000..2d748da --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscoveryContext.java @@ -0,0 +1,170 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.store.graph; + +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasObjectId; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.type.AtlasEntityType; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +public final class EntityGraphDiscoveryContext { + + /** + * Keeps track of all the entities that need to be created/updated including its child entities * + */ + private Set<AtlasEntity> rootEntities = new LinkedHashSet<>(); + + //Key is a transient id/guid + /** + * These references have been resolved using a unique identifier like guid or a qualified name etc in Atlas repository + */ + private Map<String, AtlasVertex> repositoryResolvedReferences = new LinkedHashMap<>(); + + /** + * Unresolved entity references + */ + private List<AtlasEntity> unresolvedEntityReferences = new ArrayList<>(); + + /** + * Unresolved entity id references + */ + private Set<AtlasObjectId> unresolvedIdReferences = new HashSet<>(); + + public void addRepositoryResolvedReference(AtlasObjectId id, AtlasVertex vertex) { + repositoryResolvedReferences.put(id.getGuid(), vertex); + } + + public void addUnResolvedEntityReference(AtlasEntity entity) { + this.unresolvedEntityReferences.add(entity); + } + + public void addUnResolvedIdReference(AtlasEntityType entityType, String id) { + this.unresolvedIdReferences.add(new AtlasObjectId(entityType.getTypeName(), id)); + } + + public Set<AtlasObjectId> getUnresolvedIdReferences() { + return unresolvedIdReferences; + } + + public boolean isResolved(String guid) { + return repositoryResolvedReferences.containsKey(guid); + } + + public AtlasVertex getResolvedReference(AtlasObjectId ref) { + return repositoryResolvedReferences.get(ref.getGuid()); + } + + public Map<String, AtlasVertex> getRepositoryResolvedReferences() { + return repositoryResolvedReferences; + } + + public AtlasVertex getResolvedReference(String id) { + return repositoryResolvedReferences.get(id); + } + + public List<AtlasEntity> getUnResolvedEntityReferences() { + return unresolvedEntityReferences; + } + + public void addRootEntity(AtlasEntity rootEntity) { + this.rootEntities.add(rootEntity); + } + + public Collection<AtlasEntity> getRootEntities() { + return rootEntities; + } + + public boolean removeUnResolvedEntityReference(final AtlasEntity entity) { + return unresolvedEntityReferences.remove(entity); + } + + public boolean removeUnResolvedEntityReferences(final List<AtlasEntity> entities) { + return unresolvedEntityReferences.removeAll(entities); + } + + public boolean removeUnResolvedIdReferences(final List<AtlasObjectId> entities) { + return unresolvedIdReferences.removeAll(entities); + } + + public boolean removeUnResolvedIdReference(final AtlasObjectId entity) { + return unresolvedIdReferences.remove(entity); + } + + public boolean hasUnresolvedReferences() { + return unresolvedEntityReferences.size() > 0 || unresolvedIdReferences.size() > 0; + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } else if (obj == this) { + return true; + } else if (obj.getClass() != getClass()) { + return false; + } else { + EntityGraphDiscoveryContext ctx = (EntityGraphDiscoveryContext) obj; + return Objects.equals(rootEntities, ctx.getRootEntities()) && + Objects.equals(repositoryResolvedReferences, ctx.getRepositoryResolvedReferences()) && + Objects.equals(unresolvedEntityReferences, ctx.getUnResolvedEntityReferences()) && + Objects.equals(unresolvedIdReferences, ctx.getUnresolvedIdReferences()); + } + } + + @Override + public int hashCode() { + return Objects.hash(rootEntities, repositoryResolvedReferences, unresolvedEntityReferences, unresolvedIdReferences); + } + + public StringBuilder toString(StringBuilder sb) { + if (sb == null) { + sb = new StringBuilder(); + } + + sb.append("EntityGraphDiscoveryCtx{"); + sb.append("rootEntities='").append(rootEntities).append('\''); + sb.append(", repositoryResolvedReferences=").append(repositoryResolvedReferences); + sb.append(", unresolvedEntityReferences='").append(unresolvedEntityReferences).append('\''); + sb.append(", unresolvedIdReferences='").append(unresolvedIdReferences).append('\''); + sb.append('}'); + + return sb; + } + + @Override + public String toString() { + return toString(new StringBuilder()).toString(); + } + + public void cleanUp() { + rootEntities.clear(); + unresolvedEntityReferences.clear(); + repositoryResolvedReferences.clear(); + unresolvedIdReferences.clear(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2f1cb57a/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityResolver.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityResolver.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityResolver.java new file mode 100644 index 0000000..35ddc7d --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityResolver.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.store.graph; + +import org.apache.atlas.exception.AtlasBaseException; + + +public interface EntityResolver { + + void init(EntityGraphDiscoveryContext entities) throws AtlasBaseException; + + EntityGraphDiscoveryContext resolveEntityReferences() throws AtlasBaseException; + + void cleanUp() throws AtlasBaseException; +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2f1cb57a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/ArrayVertexMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/ArrayVertexMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/ArrayVertexMapper.java new file mode 100644 index 0000000..528430c --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/ArrayVertexMapper.java @@ -0,0 +1,179 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.store.graph.v1; + +import com.google.common.base.Optional; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import org.apache.atlas.aspect.Monitored; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.typedef.AtlasStructDef; +import org.apache.atlas.repository.graph.GraphHelper; +import org.apache.atlas.repository.graphdb.AtlasEdge; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.type.AtlasArrayType; +import org.apache.atlas.type.AtlasStructType; +import org.apache.atlas.type.AtlasType; +import org.apache.commons.collections.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import static org.apache.atlas.repository.graph.GraphHelper.string; + +@Singleton +public class ArrayVertexMapper implements InstanceGraphMapper<List> { + + private static final Logger LOG = LoggerFactory.getLogger(ArrayVertexMapper.class); + + protected final DeleteHandlerV1 deleteHandler; + + protected StructVertexMapper structVertexMapper; + + @Inject + public ArrayVertexMapper(DeleteHandlerV1 deleteHandler) { + this.deleteHandler = deleteHandler; + } + + void init(StructVertexMapper structVertexMapper) { + this.structVertexMapper = structVertexMapper; + } + + @Override + public List toGraph(GraphMutationContext ctx) throws AtlasBaseException { + + if (LOG.isDebugEnabled()) { + LOG.debug("Mapping instance to vertex {} for array attribute {}", string(ctx.getReferringVertex()), ctx.getAttrType().getTypeName()); + } + + List newElements = (List) ctx.getValue(); + boolean newAttributeEmpty = (newElements == null || newElements.isEmpty()); + + AtlasArrayType arrType = (AtlasArrayType) ctx.getAttrType(); + AtlasType elementType = arrType.getElementType(); + List<Object> currentElements = getArrayElementsProperty(elementType, ctx.getReferringVertex(), ctx.getVertexPropertyKey()); + + List<Object> newElementsCreated = new ArrayList<>(); + + if (!newAttributeEmpty) { + for (int index = 0; index < newElements.size(); index++) { + + LOG.debug("Adding/updating element at position {}, current element {}, new element {}", index, + (currentElements != null && index < currentElements.size()) ? currentElements.get(index) : null, newElements.get(index)); + + Optional<AtlasEdge> existingEdge = getEdgeAt(currentElements, index, arrType.getElementType()); + + GraphMutationContext arrCtx = new GraphMutationContext.Builder(ctx.getAttribute(), + arrType.getElementType(), newElements.get(index)) + .referringVertex(ctx.getReferringVertex()) + .edge(existingEdge) + .vertexProperty(ctx.getVertexPropertyKey()).build(); + + Object newEntry = structVertexMapper.mapCollectionElementsToVertex(arrCtx); + newElementsCreated.add(newEntry); + } + } + + if (AtlasGraphUtilsV1.isReference(elementType)) { + List<AtlasEdge> additionalEdges = removeUnusedArrayEntries(ctx.getParentType(), ctx.getAttributeDef(), (List) currentElements, (List) newElementsCreated, elementType); + newElementsCreated.addAll(additionalEdges); + } + + // for dereference on way out + setArrayElementsProperty(elementType, ctx.getReferringVertex(), ctx.getVertexPropertyKey(), newElementsCreated); + return newElementsCreated; + } + + @Override + public void cleanUp() throws AtlasBaseException { + + } + + //Removes unused edges from the old collection, compared to the new collection + private List<AtlasEdge> removeUnusedArrayEntries( + AtlasStructType entityType, + AtlasStructDef.AtlasAttributeDef attributeDef, + List<AtlasEdge> currentEntries, + List<AtlasEdge> newEntries, + AtlasType entryType) throws AtlasBaseException { + if (currentEntries != null && !currentEntries.isEmpty()) { + LOG.debug("Removing unused entries from the old collection"); + if (AtlasGraphUtilsV1.isReference(entryType)) { + + Collection<AtlasEdge> edgesToRemove = CollectionUtils.subtract(currentEntries, newEntries); + + LOG.debug("Removing unused entries from the old collection - {}", edgesToRemove); + + if (!edgesToRemove.isEmpty()) { + //Remove the edges for (current edges - new edges) + List<AtlasEdge> additionalElements = new ArrayList<>(); + + for (AtlasEdge edge : edgesToRemove) { + boolean deleteChildReferences = StructVertexMapper.shouldManageChildReferences(entityType, attributeDef.getName()); + boolean deleted = deleteHandler.deleteEdgeReference(edge, entryType.getTypeCategory(), + deleteChildReferences, true); + if (!deleted) { + additionalElements.add(edge); + } + } + + return additionalElements; + } + } + } + return Collections.emptyList(); + } + + public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) { + String actualPropertyName = GraphHelper.encodePropertyKey(propertyName); + if (AtlasGraphUtilsV1.isReference(elementType)) { + return (List)instanceVertex.getListProperty(actualPropertyName, AtlasEdge.class); + } + else { + return (List)instanceVertex.getListProperty(actualPropertyName); + } + } + + private Optional<AtlasEdge> getEdgeAt(List<Object> currentElements, int index, AtlasType elemType) { + Optional<AtlasEdge> existingEdge = Optional.absent(); + if ( AtlasGraphUtilsV1.isReference(elemType) ) { + Object currentElement = (currentElements != null && index < currentElements.size()) ? + currentElements.get(index) : null; + + if ( currentElement != null) { + existingEdge = Optional.of((AtlasEdge) currentElement); + } + } + + return existingEdge; + } + + private void setArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName, List<Object> values) { + String actualPropertyName = GraphHelper.encodePropertyKey(propertyName); + if (AtlasGraphUtilsV1.isReference(elementType)) { + GraphHelper.setListPropertyFromElementIds(instanceVertex, actualPropertyName, (List) values); + } + else { + GraphHelper.setProperty(instanceVertex, actualPropertyName, values); + } + } +}
