Repository: atlas Updated Branches: refs/heads/master 75947444b -> 7e4788ed0
ATLAS-2936: Ability to store provenance type Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/7e4788ed Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/7e4788ed Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/7e4788ed Branch: refs/heads/master Commit: 7e4788ed00c6a0ef4e8c89f0486b8073a56409e4 Parents: 7594744 Author: grahamwallis <graham_wal...@uk.ibm.com> Authored: Thu Nov 1 11:17:45 2018 +0000 Committer: grahamwallis <graham_wal...@uk.ibm.com> Committed: Thu Nov 1 11:17:45 2018 +0000 ---------------------------------------------------------------------- .../org/apache/atlas/repository/Constants.java | 8 ++ .../atlas/model/instance/AtlasEntity.java | 75 ++++++++------ .../atlas/model/instance/AtlasRelationship.java | 100 +++++++++++-------- .../atlas/repository/graph/GraphHelper.java | 4 + .../graph/v2/AtlasRelationshipStoreV2.java | 4 + .../store/graph/v2/EntityGraphMapper.java | 4 + .../store/graph/v2/EntityGraphRetriever.java | 11 +- 7 files changed, 134 insertions(+), 72 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/7e4788ed/common/src/main/java/org/apache/atlas/repository/Constants.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index ed2ca4c..1cfe940 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -103,6 +103,14 @@ public final class Constants { */ public static final String IS_PROXY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "isProxy"); + /** + * The provenanceType field is used to record the provenance of an instance of an entity or relationship - this + * indicates how the instance was created. This corresponds to the InstanceProvenanceType enum defined in ODPi. + * To avoid creating a hard dependency on the ODPi class, the value is stored as an int corresponding to the + * ordinal in the ODPi enum. + */ + public static final String PROVENANCE_TYPE_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "provenanceType"); + public static final String TIMESTAMP_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "timestamp"); public static final String MODIFICATION_TIMESTAMP_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modificationTimestamp"); http://git-wip-us.apache.org/repos/asf/atlas/blob/7e4788ed/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java index 066b0d0..18018d7 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java @@ -59,30 +59,32 @@ import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ public class AtlasEntity extends AtlasStruct implements Serializable { private static final long serialVersionUID = 1L; - public static final String KEY_GUID = "guid"; - public static final String KEY_HOME_ID = "homeId"; - public static final String KEY_IS_PROXY = "isProxy"; - public static final String KEY_STATUS = "status"; - public static final String KEY_CREATED_BY = "createdBy"; - public static final String KEY_UPDATED_BY = "updatedBy"; - public static final String KEY_CREATE_TIME = "createTime"; - public static final String KEY_UPDATE_TIME = "updateTime"; - public static final String KEY_VERSION = "version"; + public static final String KEY_GUID = "guid"; + public static final String KEY_HOME_ID = "homeId"; + public static final String KEY_IS_PROXY = "isProxy"; + public static final String KEY_PROVENANCE_TYPE = "provenanceType"; + public static final String KEY_STATUS = "status"; + public static final String KEY_CREATED_BY = "createdBy"; + public static final String KEY_UPDATED_BY = "updatedBy"; + public static final String KEY_CREATE_TIME = "createTime"; + public static final String KEY_UPDATE_TIME = "updateTime"; + public static final String KEY_VERSION = "version"; /** * Status of the entity - can be active or deleted. Deleted entities are not removed from Atlas store. */ public enum Status { ACTIVE, DELETED } - private String guid = null; - private String homeId = null; - private Boolean isProxy = Boolean.FALSE; - private Status status = Status.ACTIVE; - private String createdBy = null; - private String updatedBy = null; - private Date createTime = null; - private Date updateTime = null; - private Long version = 0L; + private String guid = null; + private String homeId = null; + private Boolean isProxy = Boolean.FALSE; + private Integer provenanceType = 0; + private Status status = Status.ACTIVE; + private String createdBy = null; + private String updatedBy = null; + private Date createTime = null; + private Date updateTime = null; + private Long version = 0L; private Map<String, Object> relationshipAttributes; private List<AtlasClassification> classifications; @@ -119,15 +121,16 @@ public class AtlasEntity extends AtlasStruct implements Serializable { super(map); if (map != null) { - Object oGuid = map.get(KEY_GUID); - Object homeId = map.get(KEY_HOME_ID); - Object isProxy = map.get(KEY_IS_PROXY); - Object status = map.get(KEY_STATUS); - Object createdBy = map.get(KEY_CREATED_BY); - Object updatedBy = map.get(KEY_UPDATED_BY); - Object createTime = map.get(KEY_CREATE_TIME); - Object updateTime = map.get(KEY_UPDATE_TIME); - Object version = map.get(KEY_VERSION); + Object oGuid = map.get(KEY_GUID); + Object homeId = map.get(KEY_HOME_ID); + Object isProxy = map.get(KEY_IS_PROXY); + Object provenanceType = map.get(KEY_PROVENANCE_TYPE); + Object status = map.get(KEY_STATUS); + Object createdBy = map.get(KEY_CREATED_BY); + Object updatedBy = map.get(KEY_UPDATED_BY); + Object createTime = map.get(KEY_CREATE_TIME); + Object updateTime = map.get(KEY_UPDATE_TIME); + Object version = map.get(KEY_VERSION); if (oGuid != null) { setGuid(oGuid.toString()); @@ -144,6 +147,10 @@ public class AtlasEntity extends AtlasStruct implements Serializable { setIsProxy(Boolean.FALSE); } + if (provenanceType instanceof Number) { + setProvenanceType(((Number) version).intValue()); + } + if (status != null) { setStatus(Status.valueOf(status.toString())); } @@ -177,6 +184,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable { setGuid(other.getGuid()); setHomeId(other.getHomeId()); setIsProxy(other.isProxy()); + setProvenanceType(other.getProvenanceType()); setStatus(other.getStatus()); setCreatedBy(other.getCreatedBy()); setUpdatedBy(other.getUpdatedBy()); @@ -211,6 +219,14 @@ public class AtlasEntity extends AtlasStruct implements Serializable { this.isProxy = isProxy; } + public Integer getProvenanceType() { + return provenanceType; + } + + public void setProvenanceType(Integer provenanceType) { + this.provenanceType = provenanceType; + } + public Status getStatus() { return status; } @@ -328,6 +344,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable { setGuid(nextInternalId()); setHomeId(null); setIsProxy(Boolean.FALSE); + setProvenanceType(0); setStatus(null); setCreatedBy(null); setUpdatedBy(null); @@ -352,6 +369,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable { sb.append("guid='").append(guid).append('\''); sb.append(", homeId='").append(homeId).append('\''); sb.append(", isProxy='").append(isProxy).append('\''); + sb.append(", provenanceType=").append(provenanceType); sb.append(", status=").append(status); sb.append(", createdBy='").append(createdBy).append('\''); sb.append(", updatedBy='").append(updatedBy).append('\''); @@ -382,6 +400,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable { return Objects.equals(guid, that.guid) && Objects.equals(homeId, that.homeId) && Objects.equals(isProxy, that.isProxy) && + Objects.equals(provenanceType, that.provenanceType) && status == that.status && Objects.equals(createdBy, that.createdBy) && Objects.equals(updatedBy, that.updatedBy) && @@ -394,7 +413,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable { @Override public int hashCode() { - return Objects.hash(super.hashCode(), guid, homeId, isProxy, status, createdBy, updatedBy, createTime, updateTime, version, + return Objects.hash(super.hashCode(), guid, homeId, isProxy, provenanceType, status, createdBy, updatedBy, createTime, updateTime, version, relationshipAttributes, classifications); } http://git-wip-us.apache.org/repos/asf/atlas/blob/7e4788ed/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java index bc79dad..8681cfd 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java @@ -54,34 +54,36 @@ import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ public class AtlasRelationship extends AtlasStruct implements Serializable { private static final long serialVersionUID = 1L; - public static final String KEY_GUID = "guid"; - public static final String KEY_HOME_ID = "homeId"; - public static final String KEY_STATUS = "status"; - public static final String KEY_CREATED_BY = "createdBy"; - public static final String KEY_UPDATED_BY = "updatedBy"; - public static final String KEY_CREATE_TIME = "createTime"; - public static final String KEY_UPDATE_TIME = "updateTime"; - public static final String KEY_VERSION = "version"; - public static final String KEY_END1 = "end1"; - public static final String KEY_END2 = "end2"; - public static final String KEY_LABEL = "label"; - public static final String KEY_PROPAGATE_TAGS = "propagateTags"; + public static final String KEY_GUID = "guid"; + public static final String KEY_HOME_ID = "homeId"; + public static final String KEY_PROVENANCE_TYPE = "provenanceType"; + public static final String KEY_STATUS = "status"; + public static final String KEY_CREATED_BY = "createdBy"; + public static final String KEY_UPDATED_BY = "updatedBy"; + public static final String KEY_CREATE_TIME = "createTime"; + public static final String KEY_UPDATE_TIME = "updateTime"; + public static final String KEY_VERSION = "version"; + public static final String KEY_END1 = "end1"; + public static final String KEY_END2 = "end2"; + public static final String KEY_LABEL = "label"; + public static final String KEY_PROPAGATE_TAGS = "propagateTags"; public static final String KEY_BLOCKED_PROPAGATED_CLASSIFICATIONS = "blockedPropagatedClassifications"; public static final String KEY_PROPAGATED_CLASSIFICATIONS = "propagatedClassifications"; - private String guid = null; - private String homeId = null; - private AtlasObjectId end1 = null; - private AtlasObjectId end2 = null; - private String label = null; - private PropagateTags propagateTags = PropagateTags.NONE; - private Status status = Status.ACTIVE; - private String createdBy = null; - private String updatedBy = null; - private Date createTime = null; - private Date updateTime = null; - private Long version = 0L; + private String guid = null; + private String homeId = null; + private Integer provenanceType = null; + private AtlasObjectId end1 = null; + private AtlasObjectId end2 = null; + private String label = null; + private PropagateTags propagateTags = PropagateTags.NONE; + private Status status = Status.ACTIVE; + private String createdBy = null; + private String updatedBy = null; + private Date createTime = null; + private Date updateTime = null; + private Long version = 0L; public enum Status { ACTIVE, DELETED } @@ -110,13 +112,13 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { public AtlasRelationship(String typeName, AtlasObjectId end1, AtlasObjectId end2) { super(typeName); - init(nextInternalId(), null, end1, end2, null, null, null, null, null, null, null, 0L, null, null); + init(nextInternalId(), null, 0, end1, end2, null, null, null, null, null, null, null, 0L, null, null); } public AtlasRelationship(String typeName, AtlasObjectId end1, AtlasObjectId end2, Map<String, Object> attributes) { super(typeName, attributes); - init(nextInternalId(), null, end1, end2, null, null, null, null, null, null, null, 0L, null, null); + init(nextInternalId(), null, 0, end1, end2, null, null, null, null, null, null, null, 0L, null, null); } public AtlasRelationship(String typeName, String attrName, Object attrValue) { @@ -133,18 +135,19 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { super(map); if (map != null) { - Object oGuid = map.get(KEY_GUID); - Object homeId = map.get(KEY_HOME_ID); - Object oEnd1 = map.get(KEY_END1); - Object oEnd2 = map.get(KEY_END2); - Object label = map.get(KEY_LABEL); - Object propagateTags = map.get(KEY_PROPAGATE_TAGS); - Object status = map.get(KEY_STATUS); - Object createdBy = map.get(KEY_CREATED_BY); - Object updatedBy = map.get(KEY_UPDATED_BY); - Object createTime = map.get(KEY_CREATE_TIME); - Object updateTime = map.get(KEY_UPDATE_TIME); - Object version = map.get(KEY_VERSION); + Object oGuid = map.get(KEY_GUID); + Object homeId = map.get(KEY_HOME_ID); + Object provenanceType = map.get(KEY_PROVENANCE_TYPE); + Object oEnd1 = map.get(KEY_END1); + Object oEnd2 = map.get(KEY_END2); + Object label = map.get(KEY_LABEL); + Object propagateTags = map.get(KEY_PROPAGATE_TAGS); + Object status = map.get(KEY_STATUS); + Object createdBy = map.get(KEY_CREATED_BY); + Object updatedBy = map.get(KEY_UPDATED_BY); + Object createTime = map.get(KEY_CREATE_TIME); + Object updateTime = map.get(KEY_UPDATE_TIME); + Object version = map.get(KEY_VERSION); Object propagatedClassifications = map.get(KEY_PROPAGATED_CLASSIFICATIONS); Object blockedPropagatedClassifications = map.get(KEY_BLOCKED_PROPAGATED_CLASSIFICATIONS); @@ -157,6 +160,10 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { setHomeId(homeId.toString()); } + if (provenanceType instanceof Number) { + setProvenanceType(((Number) provenanceType).intValue()); + } + if (oEnd1 != null) { if (oEnd1 instanceof AtlasObjectId) { setEnd1((AtlasObjectId) oEnd1); @@ -235,7 +242,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { super(other); if (other != null) { - init(other.guid, other.homeId, other.end1, other.end2, other.label, other.propagateTags, other.status, other.createdBy, other.updatedBy, + init(other.guid, other.homeId, other.provenanceType, other.end1, other.end2, other.label, other.propagateTags, other.status, other.createdBy, other.updatedBy, other.createTime, other.updateTime, other.version, other.propagatedClassifications, other.blockedPropagatedClassifications); } } @@ -256,6 +263,12 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { this.homeId = homeId; } + public Integer getProvenanceType() { + return provenanceType; + } + + public void setProvenanceType(Integer provenanceType) { this.provenanceType = provenanceType; } + public Status getStatus() { return status; } @@ -341,14 +354,15 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { } private void init() { - init(nextInternalId(), null, null, null, null, null, null, null, null, null, null, 0L, null, null); + init(nextInternalId(), null, 0 ,null, null, null, null, null, null, null, null, null, 0L, null, null); } - private void init(String guid, String homeId, AtlasObjectId end1, AtlasObjectId end2, String label, PropagateTags propagateTags, + private void init(String guid, String homeId, Integer provenanceType, AtlasObjectId end1, AtlasObjectId end2, String label, PropagateTags propagateTags, Status status, String createdBy, String updatedBy, Date createTime, Date updateTime, Long version, Set<AtlasClassification> propagatedClassifications, Set<AtlasClassification> blockedPropagatedClassifications) { setGuid(guid); setHomeId(homeId); + setProvenanceType(provenanceType); setEnd1(end1); setEnd2(end2); setLabel(label); @@ -373,6 +387,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { super.toString(sb); sb.append("guid='").append(guid).append('\''); sb.append(", homeId='").append(homeId).append('\''); + sb.append(", provenanceType=").append(provenanceType); sb.append(", end1=").append(end1); sb.append(", end2=").append(end2); sb.append(", label='").append(label).append('\''); @@ -403,6 +418,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { AtlasRelationship that = (AtlasRelationship) o; return Objects.equals(guid, that.guid) && Objects.equals(homeId, that.homeId) && + Objects.equals(provenanceType, that.provenanceType) && Objects.equals(end1, that.end1) && Objects.equals(end2, that.end2) && Objects.equals(label, that.label) && @@ -419,7 +435,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { @Override public int hashCode() { - return Objects.hash(super.hashCode(), guid, homeId, end1, end2, label, propagateTags, status, createdBy, updatedBy, + return Objects.hash(super.hashCode(), guid, homeId, provenanceType, end1, end2, label, propagateTags, status, createdBy, updatedBy, createTime, updateTime, version, propagatedClassifications, blockedPropagatedClassifications); } http://git-wip-us.apache.org/repos/asf/atlas/blob/7e4788ed/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index b4189d2..45bf8dc 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -1086,6 +1086,10 @@ public final class GraphHelper { return element.getProperty(Constants.IS_PROXY_KEY, Boolean.class); } + public static Integer getProvenanceType(AtlasElement element) { + return element.getProperty(Constants.PROVENANCE_TYPE_KEY, Integer.class); + } + public static String getTypeName(AtlasElement element) { return element.getProperty(ENTITY_TYPE_PROPERTY_KEY, String.class); } http://git-wip-us.apache.org/repos/asf/atlas/blob/7e4788ed/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java index 512fff4..21617dc 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java @@ -74,6 +74,9 @@ import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY; import static org.apache.atlas.repository.Constants.RELATIONSHIP_GUID_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.VERSION_PROPERTY_KEY; +import static org.apache.atlas.repository.Constants.PROVENANCE_TYPE_KEY; + + import static org.apache.atlas.repository.graph.GraphHelper.getBlockedClassificationIds; import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEntityGuid; import static org.apache.atlas.repository.graph.GraphHelper.getClassificationName; @@ -748,6 +751,7 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore { AtlasGraphUtilsV2.setEncodedProperty(ret, ENTITY_TYPE_PROPERTY_KEY, relationship.getTypeName()); AtlasGraphUtilsV2.setEncodedProperty(ret, RELATIONSHIP_GUID_PROPERTY_KEY, guid); AtlasGraphUtilsV2.setEncodedProperty(ret, VERSION_PROPERTY_KEY, getRelationshipVersion(relationship)); + AtlasGraphUtilsV2.setEncodedProperty(ret, PROVENANCE_TYPE_KEY, relationship.getProvenanceType()); AtlasGraphUtilsV2.setEncodedProperty(ret, RELATIONSHIPTYPE_TAG_PROPAGATION_KEY, tagPropagation.name()); // blocked propagated classifications http://git-wip-us.apache.org/repos/asf/atlas/blob/7e4788ed/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index c08f4c4..0452954 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -165,6 +165,10 @@ public class EntityGraphMapper { if (entity.isProxy() != null) { AtlasGraphUtilsV2.setEncodedProperty(vertex, IS_PROXY_KEY, entity.isProxy()); } + + if (entity.getProvenanceType() != null) { + AtlasGraphUtilsV2.setEncodedProperty(vertex, PROVENANCE_TYPE_KEY, entity.getProvenanceType()); + } } public EntityMutationResponse mapAttributesAndClassifications(EntityMutationContext context, final boolean isPartialUpdate, final boolean replaceClassifications) throws AtlasBaseException { http://git-wip-us.apache.org/repos/asf/atlas/blob/7e4788ed/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index cdf69e3..c8a760b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -525,6 +525,8 @@ public final class EntityGraphRetriever { entity.setIsProxy(GraphHelper.isProxy(entityVertex)); + entity.setProvenanceType(GraphHelper.getProvenanceType(entityVertex)); + return entity; } @@ -1166,12 +1168,17 @@ public final class EntityGraphRetriever { relationship.setUpdateTime(new Date(GraphHelper.getModifiedTime(edge))); Long version = GraphHelper.getVersion(edge); - if (version == null) { version = Long.valueOf(1L); } - relationship.setVersion(version); + + Integer provenanceType = GraphHelper.getProvenanceType(edge); + if (provenanceType == null) { + provenanceType = Integer.valueOf(0); + } + relationship.setProvenanceType(provenanceType); + relationship.setStatus(GraphHelper.getEdgeStatus(edge)); AtlasVertex end1Vertex = edge.getOutVertex();