Repository: atlas Updated Branches: refs/heads/branch-1.0 41f35aaf2 -> 54932c316
ATLAS-2523 Add HomeId and allow GUIDs to be specified on creates (cherry picked from commit 72cc566a7f23bef86c00afb6c7ef136c1d01b424) Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/54932c31 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/54932c31 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/54932c31 Branch: refs/heads/branch-1.0 Commit: 54932c3163734d0d64fe37a21361e14548ca11aa Parents: 41f35aa Author: Graham Wallis <[email protected]> Authored: Fri May 25 13:35:16 2018 +0100 Committer: Sarath Subramanian <[email protected]> Committed: Fri Aug 24 15:02:14 2018 -0700 ---------------------------------------------------------------------- .../org/apache/atlas/repository/Constants.java | 12 ++++++++ .../atlas/model/instance/AtlasEntity.java | 21 +++++++++++++- .../atlas/model/instance/AtlasRelationship.java | 30 ++++++++++++++++---- .../org/apache/atlas/type/AtlasTypeUtil.java | 21 ++++++++++---- .../apache/atlas/query/RegistryBasedLookup.java | 4 ++- .../atlas/repository/graph/GraphHelper.java | 4 +++ .../graph/v2/AtlasRelationshipStoreV2.java | 4 ++- .../store/graph/v2/EntityGraphMapper.java | 4 +++ .../store/graph/v2/EntityGraphRetriever.java | 2 ++ 9 files changed, 87 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/54932c31/common/src/main/java/org/apache/atlas/repository/Constants.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index cceed76..a892551 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -81,6 +81,18 @@ public final class Constants { public static final String CREATED_BY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "createdBy"; public static final String MODIFIED_BY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "modifiedBy"; + /** + * The homeId field is used when saving into Atlas a copy of an object that is being imported from another + * repository. The homeId will be set to a String that identifies the other repository. The specific format + * of repository identifiers is domain dependent. Where it is set by Open Metadata Repository Services it will + * be a MetadataCollectionId. + * An object that is mastered by the Atlas repository, will have a null homeId field. This represents a locally + * mastered object that can be manipulated by Atlas and its applications as normal. + * An object with a non-null homeId is a copy of an object mastered by a different repository and the object + * should only be updated via the notifications and calls from Open Metadata Repository Services. + */ + public static final String HOME_ID_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "homeId"; + public static final String TIMESTAMP_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "timestamp"; public static final String MODIFICATION_TIMESTAMP_PROPERTY_KEY = http://git-wip-us.apache.org/repos/asf/atlas/blob/54932c31/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java index 7f36a10..dc2be41 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java @@ -60,6 +60,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable { private static final long serialVersionUID = 1L; public static final String KEY_GUID = "guid"; + public static final String KEY_HOME_ID = "homeId"; public static final String KEY_STATUS = "status"; public static final String KEY_CREATED_BY = "createdBy"; public static final String KEY_UPDATED_BY = "updatedBy"; @@ -73,6 +74,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable { public enum Status { ACTIVE, DELETED } private String guid = null; + private String homeId = null; private Status status = Status.ACTIVE; private String createdBy = null; private String updatedBy = null; @@ -116,6 +118,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable { if (map != null) { Object oGuid = map.get(KEY_GUID); + Object homeId = map.get(KEY_HOME_ID); Object status = map.get(KEY_STATUS); Object createdBy = map.get(KEY_CREATED_BY); Object updatedBy = map.get(KEY_UPDATED_BY); @@ -127,6 +130,10 @@ public class AtlasEntity extends AtlasStruct implements Serializable { setGuid(oGuid.toString()); } + if (homeId != null) { + setHomeId(homeId.toString()); + } + if (status != null) { setStatus(Status.valueOf(status.toString())); } @@ -158,6 +165,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable { if (other != null) { setGuid(other.getGuid()); + setHomeId(other.getHomeId()); setStatus(other.getStatus()); setCreatedBy(other.getCreatedBy()); setUpdatedBy(other.getUpdatedBy()); @@ -176,6 +184,14 @@ public class AtlasEntity extends AtlasStruct implements Serializable { this.guid = guid; } + public String getHomeId() { + return homeId; + } + + public void setHomeId(String homeId) { + this.homeId = homeId; + } + public Status getStatus() { return status; } @@ -291,6 +307,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable { private void init() { setGuid(nextInternalId()); + setHomeId(null); setStatus(null); setCreatedBy(null); setUpdatedBy(null); @@ -313,6 +330,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable { sb.append("AtlasEntity{"); super.toString(sb); sb.append("guid='").append(guid).append('\''); + sb.append(", homeId='").append(homeId).append('\''); sb.append(", status=").append(status); sb.append(", createdBy='").append(createdBy).append('\''); sb.append(", updatedBy='").append(updatedBy).append('\''); @@ -341,6 +359,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable { AtlasEntity that = (AtlasEntity) o; return Objects.equals(guid, that.guid) && + Objects.equals(homeId, that.homeId) && status == that.status && Objects.equals(createdBy, that.createdBy) && Objects.equals(updatedBy, that.updatedBy) && @@ -353,7 +372,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable { @Override public int hashCode() { - return Objects.hash(super.hashCode(), guid, status, createdBy, updatedBy, createTime, updateTime, version, + return Objects.hash(super.hashCode(), guid, homeId, status, createdBy, updatedBy, createTime, updateTime, version, relationshipAttributes, classifications); } http://git-wip-us.apache.org/repos/asf/atlas/blob/54932c31/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java index debaeef..bc79dad 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java @@ -55,6 +55,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { private static final long serialVersionUID = 1L; public static final String KEY_GUID = "guid"; + public static final String KEY_HOME_ID = "homeId"; public static final String KEY_STATUS = "status"; public static final String KEY_CREATED_BY = "createdBy"; public static final String KEY_UPDATED_BY = "updatedBy"; @@ -70,6 +71,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { public static final String KEY_PROPAGATED_CLASSIFICATIONS = "propagatedClassifications"; private String guid = null; + private String homeId = null; private AtlasObjectId end1 = null; private AtlasObjectId end2 = null; private String label = null; @@ -108,13 +110,13 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { public AtlasRelationship(String typeName, AtlasObjectId end1, AtlasObjectId end2) { super(typeName); - init(nextInternalId(), end1, end2, null, null, null, null, null, null, null, 0L, null, null); + init(nextInternalId(), null, end1, end2, null, null, null, null, null, null, null, 0L, null, null); } public AtlasRelationship(String typeName, AtlasObjectId end1, AtlasObjectId end2, Map<String, Object> attributes) { super(typeName, attributes); - init(nextInternalId(), end1, end2, null, null, null, null, null, null, null, 0L, null, null); + init(nextInternalId(), null, end1, end2, null, null, null, null, null, null, null, 0L, null, null); } public AtlasRelationship(String typeName, String attrName, Object attrValue) { @@ -132,6 +134,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { if (map != null) { Object oGuid = map.get(KEY_GUID); + Object homeId = map.get(KEY_HOME_ID); Object oEnd1 = map.get(KEY_END1); Object oEnd2 = map.get(KEY_END2); Object label = map.get(KEY_LABEL); @@ -150,6 +153,10 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { setGuid(oGuid.toString()); } + if (homeId != null) { + setHomeId(homeId.toString()); + } + if (oEnd1 != null) { if (oEnd1 instanceof AtlasObjectId) { setEnd1((AtlasObjectId) oEnd1); @@ -228,7 +235,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { super(other); if (other != null) { - init(other.guid, other.end1, other.end2, other.label, other.propagateTags, other.status, other.createdBy, other.updatedBy, + init(other.guid, other.homeId, other.end1, other.end2, other.label, other.propagateTags, other.status, other.createdBy, other.updatedBy, other.createTime, other.updateTime, other.version, other.propagatedClassifications, other.blockedPropagatedClassifications); } } @@ -241,6 +248,14 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { this.guid = guid; } + public String getHomeId() { + return homeId; + } + + public void setHomeId(String homeId) { + this.homeId = homeId; + } + public Status getStatus() { return status; } @@ -326,13 +341,14 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { } private void init() { - init(nextInternalId(), null, null, null, null, null, null, null, null, null, 0L, null, null); + init(nextInternalId(), null, null, null, null, null, null, null, null, null, null, 0L, null, null); } - private void init(String guid, AtlasObjectId end1, AtlasObjectId end2, String label, PropagateTags propagateTags, + private void init(String guid, String homeId, AtlasObjectId end1, AtlasObjectId end2, String label, PropagateTags propagateTags, Status status, String createdBy, String updatedBy, Date createTime, Date updateTime, Long version, Set<AtlasClassification> propagatedClassifications, Set<AtlasClassification> blockedPropagatedClassifications) { setGuid(guid); + setHomeId(homeId); setEnd1(end1); setEnd2(end2); setLabel(label); @@ -356,6 +372,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { sb.append("AtlasRelationship{"); super.toString(sb); sb.append("guid='").append(guid).append('\''); + sb.append(", homeId='").append(homeId).append('\''); sb.append(", end1=").append(end1); sb.append(", end2=").append(end2); sb.append(", label='").append(label).append('\''); @@ -385,6 +402,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { AtlasRelationship that = (AtlasRelationship) o; return Objects.equals(guid, that.guid) && + Objects.equals(homeId, that.homeId) && Objects.equals(end1, that.end1) && Objects.equals(end2, that.end2) && Objects.equals(label, that.label) && @@ -401,7 +419,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { @Override public int hashCode() { - return Objects.hash(super.hashCode(), guid, end1, end2, label, propagateTags, status, createdBy, updatedBy, + return Objects.hash(super.hashCode(), guid, homeId, end1, end2, label, propagateTags, status, createdBy, updatedBy, createTime, updateTime, version, propagatedClassifications, blockedPropagatedClassifications); } http://git-wip-us.apache.org/repos/asf/atlas/blob/54932c31/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java b/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java index 6c0fdbf..e6c17af 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java @@ -55,6 +55,7 @@ import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.*; * Utility methods for AtlasType/AtlasTypeDef. */ public class AtlasTypeUtil { + private static final Set<String> ATLAS_BUILTIN_TYPENAMES = new HashSet<>(); private static final String NAME_REGEX = "[a-zA-Z][a-zA-Z0-9_ ]*"; private static final String TRAIT_NAME_REGEX = "[a-zA-Z][a-zA-Z0-9_ .]*"; @@ -380,13 +381,21 @@ public class AtlasTypeUtil { } public static boolean isAssignedGuid(String guid) { + /** + * The rule for whether a GUID is 'assigned' is that it must always be non-null, non-empty + * and must not start with a '-' character, because in Atlas the '-' prefix character + * signifies an Atlas 'unassigned' GUID. There are no other GUID formatting constraints. + * + * An object from a remote repository can be saved into Atlas with its existing (external) GUID + * if that GUID conforms to the same 3 conditions. If, in future, it is required to save objects from + * a remote repository that assigns GUIDs that can start with the '-' character, then it will be + * necessary to enhance this isAssignedGUID() method to accepts and check the object's homeId, such + * that if homeId is not null (the object is from a remote repository), then the '-' prefix constraint + * is relaxed. Such a change would require a pervasive change to Atlas classes and therefore should + * only be implemented if it is found to be necessary. + */ if (guid != null) { - try { - UUID.fromString(guid); - return true; - } catch (IllegalArgumentException e) { - // ignore - } + return guid != null && guid.length() > 0 && guid.charAt(0) != '-'; } return false; } http://git-wip-us.apache.org/repos/asf/atlas/blob/54932c31/repository/src/main/java/org/apache/atlas/query/RegistryBasedLookup.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/query/RegistryBasedLookup.java b/repository/src/main/java/org/apache/atlas/query/RegistryBasedLookup.java index d684143..e0fb4d8 100644 --- a/repository/src/main/java/org/apache/atlas/query/RegistryBasedLookup.java +++ b/repository/src/main/java/org/apache/atlas/query/RegistryBasedLookup.java @@ -39,7 +39,9 @@ class RegistryBasedLookup implements Lookup { Constants.CREATED_BY_KEY, Constants.STATE_PROPERTY_KEY, Constants.TIMESTAMP_PROPERTY_KEY, - Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY)); + Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, + Constants.HOME_ID_KEY + )); private static final Map<String, String> NUMERIC_ATTRIBUTES = new HashMap<String, String>() {{ put(AtlasBaseTypeDef.ATLAS_TYPE_SHORT, ""); http://git-wip-us.apache.org/repos/asf/atlas/blob/54932c31/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index 3079603..50c3cdd 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -1183,6 +1183,10 @@ public final class GraphHelper { return element.<String>getProperty(Constants.GUID_PROPERTY_KEY, String.class); } + public static String getHomeId(AtlasElement element) { + return element.getProperty(Constants.HOME_ID_KEY, String.class); + } + public static String getTypeName(AtlasElement element) { return element.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, String.class); } http://git-wip-us.apache.org/repos/asf/atlas/blob/54932c31/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java index f52b3ee..5372d9b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java @@ -737,7 +737,9 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore { // map additional properties to relationship edge if (ret != null) { - final String guid = UUID.randomUUID().toString(); + // Accept a valid (assigned) guid from the supplied relationship, or generate one. + String relationshipGuid = relationship.getGuid(); + final String guid = AtlasTypeUtil.isAssignedGuid(relationshipGuid) ? relationshipGuid : UUID.randomUUID().toString(); AtlasGraphUtilsV2.setProperty(ret, Constants.ENTITY_TYPE_PROPERTY_KEY, relationship.getTypeName()); AtlasGraphUtilsV2.setProperty(ret, Constants.RELATIONSHIP_GUID_PROPERTY_KEY, guid); http://git-wip-us.apache.org/repos/asf/atlas/blob/54932c31/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 9b533b7..18a33be 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -175,6 +175,10 @@ public class EntityGraphMapper { if (StringUtils.isNotEmpty(entity.getUpdatedBy())) { AtlasGraphUtilsV2.setProperty(vertex, Constants.MODIFIED_BY_KEY, entity.getUpdatedBy()); } + + if (StringUtils.isNotEmpty(entity.getHomeId())) { + AtlasGraphUtilsV2.setProperty(vertex, Constants.HOME_ID_KEY, entity.getHomeId()); + } } public EntityMutationResponse mapAttributesAndClassifications(EntityMutationContext context, final boolean isPartialUpdate, final boolean replaceClassifications) throws AtlasBaseException { http://git-wip-us.apache.org/repos/asf/atlas/blob/54932c31/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 8d58bdd..34109b3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -511,6 +511,8 @@ public final class EntityGraphRetriever { entity.setCreateTime(new Date(GraphHelper.getCreatedTime(entityVertex))); entity.setUpdateTime(new Date(GraphHelper.getModifiedTime(entityVertex))); + entity.setHomeId(GraphHelper.getHomeId(entityVertex)); + return entity; }
