ATLAS-1959: Enhance relationship attributes to support cardinality mappings
Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/02e4e86b Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/02e4e86b Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/02e4e86b Branch: refs/heads/feature-odf Commit: 02e4e86b5adeb52044ba09a978c8ccbb2200d67d Parents: f74e43c Author: Sarath Subramanian <[email protected]> Authored: Wed Jul 19 12:05:13 2017 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Wed Jul 19 18:39:14 2017 -0700 ---------------------------------------------------------------------- addons/models/0010-base_model.json | 12 +- addons/models/0030-hive_model.json | 36 +- addons/models/0050-falcon_model.json | 18 +- addons/models/0060-hbase_model.json | 16 +- addons/models/0080-storm_model.json | 6 +- .../atlas/model/instance/AtlasEntity.java | 14 +- .../atlas/model/instance/AtlasRelationship.java | 2 - .../model/typedef/AtlasRelationshipDef.java | 2 + .../model/typedef/AtlasRelationshipEndDef.java | 25 +- .../org/apache/atlas/type/AtlasEntityType.java | 12 + .../atlas/type/AtlasRelationshipType.java | 55 ++- .../org/apache/atlas/type/AtlasStructType.java | 32 +- .../apache/atlas/TestRelationshipUtilsV2.java | 267 +++++++++++ .../test/java/org/apache/atlas/TestUtilsV2.java | 1 - .../atlas/repository/graph/GraphHelper.java | 86 +++- .../graph/v1/AtlasEntityGraphDiscoveryV1.java | 18 +- .../store/graph/v1/AtlasGraphUtilsV1.java | 2 +- .../graph/v1/AtlasRelationshipStoreV1.java | 156 ++++--- .../store/graph/v1/DeleteHandlerV1.java | 33 +- .../store/graph/v1/EntityGraphMapper.java | 240 ++++++++-- .../store/graph/v1/EntityGraphRetriever.java | 235 +++++----- .../repository/impexp/ImportServiceTest.java | 1 + .../impexp/ZipFileResourceTestUtils.java | 3 + .../AtlasRelationshipStoreHardDeleteV1Test.java | 54 +++ .../AtlasRelationshipStoreSoftDeleteV1Test.java | 55 +++ .../graph/v1/AtlasRelationshipStoreV1Test.java | 437 ++++++++++++++++++- 26 files changed, 1489 insertions(+), 329 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/addons/models/0010-base_model.json ---------------------------------------------------------------------- diff --git a/addons/models/0010-base_model.json b/addons/models/0010-base_model.json index 303f379..20996a2 100644 --- a/addons/models/0010-base_model.json +++ b/addons/models/0010-base_model.json @@ -102,15 +102,15 @@ "endDef1": { "type": "DataSet", "name": "sourceToProcesses", - "isContainer": "false", + "isContainer": false, "cardinality": "SET" }, "endDef2": { "type": "Process", "name": "inputs", - "isContainer": "true", + "isContainer": true, "cardinality": "SET", - "legacyLabel": "__Process.inputs" + "isLegacyAttribute": true }, "propagateTags": "NONE" }, @@ -121,14 +121,14 @@ "endDef1": { "type": "Process", "name": "outputs", - "isContainer": "true", + "isContainer": true, "cardinality": "SET", - "legacyLabel": "__Process.outputs" + "isLegacyAttribute": true }, "endDef2": { "type": "DataSet", "name": "sinkFromProcesses", - "isContainer": "false", + "isContainer": false, "cardinality": "SET" }, "propagateTags": "NONE" http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/addons/models/0030-hive_model.json ---------------------------------------------------------------------- diff --git a/addons/models/0030-hive_model.json b/addons/models/0030-hive_model.json index a795f0f..f47a7b9 100644 --- a/addons/models/0030-hive_model.json +++ b/addons/models/0030-hive_model.json @@ -530,15 +530,15 @@ "endDef1": { "type": "hive_db", "name": "tables", - "isContainer": "true", + "isContainer": true, "cardinality": "SET" }, "endDef2": { "type": "hive_table", "name": "db", - "isContainer": "false", + "isContainer": false, "cardinality": "SINGLE", - "legacyLabel": "__hive_table.db" + "isLegacyAttribute": true }, "propagateTags": "ONE_TO_TWO" }, @@ -549,16 +549,16 @@ "endDef1": { "type": "hive_table", "name": "columns", - "isContainer": "true", + "isContainer": true, "cardinality": "SET", - "legacyLabel": "__hive_table.columns" + "isLegacyAttribute": true }, "endDef2": { "type": "hive_column", "name": "table", - "isContainer": "false", + "isContainer": false, "cardinality": "SINGLE", - "legacyLabel": "__hive_column.table" + "isLegacyAttribute": true }, "propagateTags": "ONE_TO_TWO" }, @@ -569,16 +569,16 @@ "endDef1": { "type": "hive_table", "name": "partitionKeys", - "isContainer": "true", + "isContainer": true, "cardinality": "SET", - "legacyLabel": "__hive_table.partitionKeys" + "isLegacyAttribute": true }, "endDef2": { "type": "hive_column", "name": "table", - "isContainer": "false", + "isContainer": false, "cardinality": "SINGLE", - "legacyLabel": "__hive_column.table" + "isLegacyAttribute": true }, "propagateTags": "ONE_TO_TWO" }, @@ -589,16 +589,16 @@ "endDef1": { "type": "hive_table", "name": "sd", - "isContainer": "true", + "isContainer": true, "cardinality": "SINGLE", - "legacyLabel": "__hive_table.sd" + "isLegacyAttribute": true }, "endDef2": { "type": "hive_storagedesc", "name": "table", - "isContainer": "false", + "isContainer": false, "cardinality": "SINGLE", - "legacyLabel": "__hive_storagedesc.table" + "isLegacyAttribute": true }, "propagateTags": "ONE_TO_TWO" }, @@ -609,15 +609,15 @@ "endDef1": { "type": "hive_process", "name": "columnLineages", - "isContainer": "true", + "isContainer": true, "cardinality": "SET" }, "endDef2": { "type": "hive_column_lineage", "name": "query", - "isContainer": "false", + "isContainer": false, "cardinality": "SINGLE", - "legacyLabel": "__hive_column_lineage.query" + "isLegacyAttribute": true }, "propagateTags": "NONE" } http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/addons/models/0050-falcon_model.json ---------------------------------------------------------------------- diff --git a/addons/models/0050-falcon_model.json b/addons/models/0050-falcon_model.json index 7755fa8..4fa4604 100644 --- a/addons/models/0050-falcon_model.json +++ b/addons/models/0050-falcon_model.json @@ -152,14 +152,14 @@ "endDef1": { "type": "falcon_feed", "name": "stored-in", - "isContainer": "false", + "isContainer": false, "cardinality": "SINGLE", - "legacyLabel": "__falcon_feed.stored-in" + "isLegacyAttribute": true }, "endDef2": { "type": "falcon_cluster", "name": "feeds", - "isContainer": "true", + "isContainer": true, "cardinality": "SET" }, "propagateTags": "NONE" @@ -171,15 +171,15 @@ "endDef1": { "type": "falcon_cluster", "name": "processes", - "isContainer": "true", + "isContainer": true, "cardinality": "SET" }, "endDef2": { "type": "falcon_process", "name": "runs-on", - "isContainer": "false", + "isContainer": false, "cardinality": "SINGLE", - "legacyLabel": "__falcon_process.runs-on" + "isLegacyAttribute": true }, "propagateTags": "NONE" }, @@ -190,15 +190,15 @@ "endDef1": { "type": "falcon_cluster", "name": "feedCreations", - "isContainer": "true", + "isContainer": true, "cardinality": "SET" }, "endDef2": { "type": "falcon_feed_creation", "name": "stored-in", - "isContainer": "false", + "isContainer": false, "cardinality": "SINGLE", - "legacyLabel": "__falcon_feed_creation.stored-in" + "isLegacyAttribute": true }, "propagateTags": "NONE" } http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/addons/models/0060-hbase_model.json ---------------------------------------------------------------------- diff --git a/addons/models/0060-hbase_model.json b/addons/models/0060-hbase_model.json index 1d264df..3e46e06 100644 --- a/addons/models/0060-hbase_model.json +++ b/addons/models/0060-hbase_model.json @@ -105,16 +105,16 @@ "endDef1": { "type": "hbase_table", "name": "column_families", - "isContainer": "true", + "isContainer": true, "cardinality": "SET", - "legacyLabel": "__hbase_table.column_families" + "isLegacyAttribute": true }, "endDef2": { "type": "hbase_column_family", "name": "table", - "isContainer": "false", + "isContainer": false, "cardinality": "SINGLE", - "legacyLabel": "__hbase_column_family.table" + "isLegacyAttribute": true }, "propagateTags": "ONE_TO_TWO" }, @@ -125,16 +125,16 @@ "endDef1": { "type": "hbase_column_family", "name": "columns", - "isContainer": "true", + "isContainer": true, "cardinality": "SET", - "legacyLabel": "__hbase_column_family.columns" + "isLegacyAttribute": true }, "endDef2": { "type": "hbase_column", "name": "column_family", - "isContainer": "false", + "isContainer": false, "cardinality": "SINGLE", - "legacyLabel": "__hbase_column.column_family" + "isLegacyAttribute": true }, "propagateTags": "ONE_TO_TWO" } http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/addons/models/0080-storm_model.json ---------------------------------------------------------------------- diff --git a/addons/models/0080-storm_model.json b/addons/models/0080-storm_model.json index 25360ff..b008c7a 100644 --- a/addons/models/0080-storm_model.json +++ b/addons/models/0080-storm_model.json @@ -151,14 +151,14 @@ "endDef1": { "type": "storm_topology", "name": "nodes", - "isContainer": "false", + "isContainer": false, "cardinality": "SET", - "legacyLabel": "__storm_topology.nodes" + "isLegacyAttribute": true }, "endDef2": { "type": "storm_node", "name": "topolgies", - "isContainer": "false", + "isContainer": false, "cardinality": "SET" }, "propagateTags": "NONE" http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java index 68da6af..365e548 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java @@ -177,7 +177,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable { this.relationshipAttributes = relationshipAttributes; } - public void addRelationshipAttribute(String name, Object value) { + public void setRelationshipAttribute(String name, Object value) { Map<String, Object> r = this.relationshipAttributes; if (r != null) { @@ -190,6 +190,18 @@ public class AtlasEntity extends AtlasStruct implements Serializable { } } + public Object getRelationshipAttribute(String name) { + Map<String, Object> a = this.relationshipAttributes; + + return a != null ? a.get(name) : null; + } + + public boolean hasRelationshipAttribute(String name) { + Map<String, Object> r = this.relationshipAttributes; + + return r != null ? r.containsKey(name) : false; + } + public List<AtlasClassification> getClassifications() { return classifications; } public void setClassifications(List<AtlasClassification> classifications) { this.classifications = classifications; } http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java index 2de9bdf..4188371 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java @@ -176,8 +176,6 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { return "-" + Long.toString(s_nextId.getAndIncrement()); } - public String getRelationshipLabel() { return "r:" + super.getTypeName(); } - private void init() { init(nextInternalId(), null, null, null, null, null, null, null, null, 0L); } http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipDef.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipDef.java index fc820d4..c17e875 100644 --- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipDef.java +++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipDef.java @@ -219,6 +219,8 @@ public class AtlasRelationshipDef extends AtlasStructDef implements java.io.Seri return this.endDef2; } + public String getRelationshipLabel() { return "r:" + super.getName(); } + public AtlasRelationshipDef(AtlasRelationshipDef other) throws AtlasBaseException { super(other); http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipEndDef.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipEndDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipEndDef.java index f80ea89..01e5ce7 100644 --- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipEndDef.java +++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipEndDef.java @@ -18,7 +18,6 @@ package org.apache.atlas.model.typedef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality; -import org.apache.commons.lang.StringUtils; import org.codehaus.jackson.annotate.JsonAutoDetect; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.map.annotate.JsonSerialize; @@ -61,9 +60,9 @@ public class AtlasRelationshipEndDef implements Serializable { */ private Cardinality cardinality; /** - * legacy edge label name of the endpoint + * When set this indicates that this end is is a legacy attribute */ - private String legacyLabel; + private boolean isLegacyAttribute; /** * Base constructor @@ -97,15 +96,15 @@ public class AtlasRelationshipEndDef implements Serializable { * - whether the end is a container or not */ public AtlasRelationshipEndDef(String typeName, String name, Cardinality cardinality, boolean isContainer) { - this(typeName, name, cardinality, isContainer, null); + this(typeName, name, cardinality, isContainer, false); } - public AtlasRelationshipEndDef(String typeName, String name, Cardinality cardinality, boolean isContainer, String legacyLabel) { + public AtlasRelationshipEndDef(String typeName, String name, Cardinality cardinality, boolean isContainer, boolean isLegacyAttribute) { setType(typeName); setName(name); setCardinality(cardinality); setIsContainer(isContainer); - setLegacyLabel(legacyLabel); + setIsLegacyAttribute(isLegacyAttribute); } /** @@ -118,7 +117,7 @@ public class AtlasRelationshipEndDef implements Serializable { setName(other.getName()); setIsContainer(other.getIsContainer()); setCardinality(other.getCardinality()); - setLegacyLabel(other.getLegacyLabel()); + setIsLegacyAttribute(other.isLegacyAttribute); } } @@ -166,11 +165,9 @@ public class AtlasRelationshipEndDef implements Serializable { return this.cardinality; } - public String getLegacyLabel() { return legacyLabel; } + public boolean getIsLegacyAttribute() { return isLegacyAttribute; } - public void setLegacyLabel(String legacyLabel) { this.legacyLabel = legacyLabel; } - - public boolean hasLegacyRelation() { return StringUtils.isNotEmpty(getLegacyLabel()) ? true : false; } + public void setIsLegacyAttribute(boolean legacyAttribute) { isLegacyAttribute = legacyAttribute; } public StringBuilder toString(StringBuilder sb) { if (sb == null) { @@ -182,7 +179,7 @@ public class AtlasRelationshipEndDef implements Serializable { sb.append(", name==>'").append(name).append('\''); sb.append(", isContainer==>'").append(isContainer).append('\''); sb.append(", cardinality==>'").append(cardinality).append('\''); - sb.append(", legacyLabel==>'").append(legacyLabel).append('\''); + sb.append(", isLegacyAttribute==>'").append(isLegacyAttribute).append('\''); sb.append('}'); return sb; @@ -200,12 +197,12 @@ public class AtlasRelationshipEndDef implements Serializable { Objects.equals(name, that.name) && isContainer == that.isContainer && cardinality == that.cardinality && - Objects.equals(legacyLabel, that.legacyLabel); + isLegacyAttribute == that.isLegacyAttribute; } @Override public int hashCode() { - return Objects.hash(type, getName(), isContainer, cardinality, legacyLabel); + return Objects.hash(type, getName(), isContainer, cardinality, isLegacyAttribute); } @Override http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java index e94dd19..e3005ee 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java @@ -191,6 +191,8 @@ public class AtlasEntityType extends AtlasStructType { public Map<String, AtlasAttribute> getRelationshipAttributes() { return relationshipAttributes; } + public AtlasAttribute getRelationshipAttribute(String attributeName) { return relationshipAttributes.get(attributeName); } + // this method should be called from AtlasRelationshipType.resolveReferencesPhase2() void addRelationshipAttribute(String attributeName, AtlasAttribute attribute) { relationshipAttributes.put(attributeName, attribute); @@ -220,6 +222,16 @@ public class AtlasEntityType extends AtlasStructType { return relationshipAttributes.containsKey(attributeName); } + public String getQualifiedAttributeName(String attrName) throws AtlasBaseException { + if (allAttributes.containsKey(attrName)) { + return allAttributes.get(attrName).getQualifiedName(); + } else if (relationshipAttributes.containsKey(attrName)) { + return relationshipAttributes.get(attrName).getQualifiedName(); + } + + throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attrName, entityDef.getName()); + } + @Override public AtlasEntity createDefaultValue() { AtlasEntity ret = new AtlasEntity(entityDef.getName()); http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java b/intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java index 841b66f..934dffc 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java @@ -30,6 +30,10 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection; +import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN; +import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.OUT; + /** * class that implements behaviour of an relationship-type. */ @@ -97,19 +101,41 @@ public class AtlasRelationshipType extends AtlasStructType { // if legacyLabel is not specified at both ends, use relationshipDef name as relationship label. // if legacyLabel is specified in any one end, use it as the relationship label for both ends (legacy case). // if legacyLabel is specified at both ends use the respective end's legacyLabel as relationship label (legacy case). - if (!endDef1.hasLegacyRelation() && !endDef2.hasLegacyRelation()) { - relationshipLabel = relationshipDef.getName(); - - } else if (endDef1.hasLegacyRelation() && !endDef2.hasLegacyRelation()) { - relationshipLabel = endDef1.getLegacyLabel(); - - } else if (!endDef1.hasLegacyRelation() && endDef2.hasLegacyRelation()) { - relationshipLabel = endDef2.getLegacyLabel(); + if (!endDef1.getIsLegacyAttribute() && !endDef2.getIsLegacyAttribute()) { + relationshipLabel = relationshipDef.getRelationshipLabel(); + } else if (endDef1.getIsLegacyAttribute() && !endDef2.getIsLegacyAttribute()) { + relationshipLabel = getLegacyEdgeLabel(end1Type, endDef1.getName()); + } else if (!endDef1.getIsLegacyAttribute() && endDef2.getIsLegacyAttribute()) { + relationshipLabel = getLegacyEdgeLabel(end2Type, endDef2.getName()); } addRelationshipAttributeToEndType(endDef1, end1Type, end2Type.getTypeName(), typeRegistry, relationshipLabel); addRelationshipAttributeToEndType(endDef2, end2Type, end1Type.getTypeName(), typeRegistry, relationshipLabel); + + // add relationship edge direction information + addRelationshipEdgeDirection(); + } + + private void addRelationshipEdgeDirection() { + AtlasRelationshipEndDef endDef1 = relationshipDef.getEndDef1(); + AtlasRelationshipEndDef endDef2 = relationshipDef.getEndDef2(); + AtlasAttribute end1Attribute = end1Type.getRelationshipAttribute(endDef1.getName()); + AtlasAttribute end2Attribute = end2Type.getRelationshipAttribute(endDef2.getName()); + + //default relationship edge direction is end1 (out) -> end2 (in) + AtlasRelationshipEdgeDirection end1Direction = OUT; + AtlasRelationshipEdgeDirection end2Direction = IN; + + if (endDef1.getIsLegacyAttribute() && endDef2.getIsLegacyAttribute()) { + end2Direction = OUT; + } else if (!endDef1.getIsLegacyAttribute() && endDef2.getIsLegacyAttribute()) { + end1Direction = IN; + end2Direction = OUT; + } + + end1Attribute.setRelationshipEdgeDirection(end1Direction); + end2Attribute.setRelationshipEdgeDirection(end2Direction); } @Override @@ -229,7 +255,7 @@ public class AtlasRelationshipType extends AtlasStructType { // if relationshipLabel is null, then legacyLabel is mentioned at both ends, // use the respective end's legacyLabel as relationshipLabel if (relationshipLabel == null) { - relationshipLabel = endDef.getLegacyLabel(); + relationshipLabel = getLegacyEdgeLabel(entityType, attrName); } if (attribute == null) { //attr doesn't exist in type - is a new relationship attribute @@ -251,4 +277,15 @@ public class AtlasRelationshipType extends AtlasStructType { entityType.addRelationshipAttributeType(attrName, this); } + + private String getLegacyEdgeLabel(AtlasEntityType entityType, String attributeName) { + String ret = null; + AtlasAttribute attribute = entityType.getAttribute(attributeName); + + if (attribute != null) { + ret = "__" + attribute.getQualifiedName(); + } + + return ret; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java index f97d767..b390a97 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java @@ -601,15 +601,16 @@ public class AtlasStructType extends AtlasType { } public static class AtlasAttribute { - private final AtlasStructType definedInType; - private final AtlasType attributeType; - private final AtlasAttributeDef attributeDef; - private final String qualifiedName; - private final String vertexPropertyName; - private final boolean isOwnedRef; - private final String inverseRefAttributeName; - private AtlasAttribute inverseRefAttribute; - private String relationshipEdgeLabel; + private final AtlasStructType definedInType; + private final AtlasType attributeType; + private final AtlasAttributeDef attributeDef; + private final String qualifiedName; + private final String vertexPropertyName; + private final boolean isOwnedRef; + private final String inverseRefAttributeName; + private AtlasAttribute inverseRefAttribute; + private String relationshipEdgeLabel; + private AtlasRelationshipEdgeDirection relationshipEdgeDirection; public AtlasAttribute(AtlasStructType definedInType, AtlasAttributeDef attrDef, AtlasType attributeType, String relationshipLabel) { this.definedInType = definedInType; @@ -637,8 +638,9 @@ public class AtlasStructType extends AtlasType { } } - this.isOwnedRef = isOwnedRef; - this.inverseRefAttributeName = inverseRefAttribute; + this.isOwnedRef = isOwnedRef; + this.inverseRefAttributeName = inverseRefAttribute; + this.relationshipEdgeDirection = AtlasRelationshipEdgeDirection.OUT; } public AtlasAttribute(AtlasStructType definedInType, AtlasAttributeDef attrDef, AtlasType attributeType) { @@ -677,6 +679,12 @@ public class AtlasStructType extends AtlasType { public void setRelationshipEdgeLabel(String relationshipEdgeLabel) { this.relationshipEdgeLabel = relationshipEdgeLabel; } + public AtlasRelationshipEdgeDirection getRelationshipEdgeDirection() { return relationshipEdgeDirection; } + + public void setRelationshipEdgeDirection(AtlasRelationshipEdgeDirection relationshipEdgeDirection) { + this.relationshipEdgeDirection = relationshipEdgeDirection; + } + public static String getEdgeLabel(String property) { return "__" + property; } @@ -721,5 +729,7 @@ public class AtlasStructType extends AtlasType { new String[] { "$", "_d" }, new String[] { "%", "_p" }, }; + + public enum AtlasRelationshipEdgeDirection { IN, OUT } } } http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/intg/src/test/java/org/apache/atlas/TestRelationshipUtilsV2.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/TestRelationshipUtilsV2.java b/intg/src/test/java/org/apache/atlas/TestRelationshipUtilsV2.java new file mode 100755 index 0000000..98be2b8 --- /dev/null +++ b/intg/src/test/java/org/apache/atlas/TestRelationshipUtilsV2.java @@ -0,0 +1,267 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; +import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.model.typedef.AtlasBaseTypeDef; +import org.apache.atlas.model.typedef.AtlasClassificationDef; +import org.apache.atlas.model.typedef.AtlasEntityDef; +import org.apache.atlas.model.typedef.AtlasEnumDef; +import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumElementDef; +import org.apache.atlas.model.typedef.AtlasRelationshipDef; +import org.apache.atlas.model.typedef.AtlasRelationshipEndDef; +import org.apache.atlas.model.typedef.AtlasStructDef; +import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.atlas.type.AtlasTypeUtil; +import org.apache.commons.lang.StringUtils; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.getArrayTypeName; +import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.getMapTypeName; +import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.ONE_TO_TWO; +import static org.apache.atlas.model.typedef.AtlasRelationshipDef.RelationshipCategory.AGGREGATION; +import static org.apache.atlas.model.typedef.AtlasRelationshipDef.RelationshipCategory.ASSOCIATION; +import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.SET; +import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE; +import static org.apache.atlas.type.AtlasTypeUtil.createClassTypeDef; +import static org.apache.atlas.type.AtlasTypeUtil.createOptionalAttrDef; +import static org.apache.atlas.type.AtlasTypeUtil.createRequiredAttrDef; +import static org.apache.atlas.type.AtlasTypeUtil.createStructTypeDef; +import static org.apache.atlas.type.AtlasTypeUtil.createTraitTypeDef; +import static org.apache.atlas.type.AtlasTypeUtil.createUniqueRequiredAttrDef; +import static org.apache.atlas.type.AtlasTypeUtil.getAtlasObjectId; + +/** + * Test utility class for relationship. + */ +public final class TestRelationshipUtilsV2 { + + public static final String ORG_LEVEL_TYPE = "OrgLevel"; + public static final String SECURITY_CLEARANCE_TYPE = "SecurityClearance"; + public static final String ADDRESS_TYPE = "Address"; + public static final String PERSON_TYPE = "Person"; + public static final String MANAGER_TYPE = "Manager"; + public static final String DEPARTMENT_TYPE = "Department"; + public static final String EMPLOYEE_TYPE = "Employee"; + public static final String EMPLOYEE_DEPARTMENT_TYPE = "EmployeeDepartment"; + public static final String EMPLOYEE_MANAGER_TYPE = "EmployeeManager"; + public static final String EMPLOYEE_MENTOR_TYPE = "EmployeeMentor"; + public static final String TYPE_A = "A"; + public static final String TYPE_B = "B"; + public static final String DEFAULT_VERSION = "1.0"; + + + private TestRelationshipUtilsV2() { } + + public static AtlasTypesDef getDepartmentEmployeeTypes() throws AtlasBaseException { + + /******* Person Type *******/ + AtlasEntityDef personType = createClassTypeDef(PERSON_TYPE, description(PERSON_TYPE), superType(null), + createUniqueRequiredAttrDef("name", "string"), + createOptionalAttrDef("address", ADDRESS_TYPE), + createOptionalAttrDef("birthday", "date"), + createOptionalAttrDef("hasPets", "boolean"), + createOptionalAttrDef("numberOfCars", "byte"), + createOptionalAttrDef("houseNumber", "short"), + createOptionalAttrDef("carMileage", "int"), + createOptionalAttrDef("age", "float"), + createOptionalAttrDef("numberOfStarsEstimate", "biginteger"), + createOptionalAttrDef("approximationOfPi", "bigdecimal")); + /******* Employee Type *******/ + AtlasEntityDef employeeType = createClassTypeDef(EMPLOYEE_TYPE, description(EMPLOYEE_TYPE), superType(PERSON_TYPE), + createOptionalAttrDef("orgLevel", ORG_LEVEL_TYPE), + createOptionalAttrDef("shares", "long"), + createOptionalAttrDef("salary", "double")); + /******* Department Type *******/ + AtlasEntityDef departmentType = createClassTypeDef(DEPARTMENT_TYPE, description(DEPARTMENT_TYPE), superType(null), + createUniqueRequiredAttrDef("name", "string")); + /******* Manager Type *******/ + AtlasEntityDef managerType = createClassTypeDef(MANAGER_TYPE, description(MANAGER_TYPE), superType(EMPLOYEE_TYPE)); + /******* Address Type *******/ + AtlasStructDef addressType = createStructTypeDef(ADDRESS_TYPE, description(ADDRESS_TYPE), + createRequiredAttrDef("street", "string"), + createRequiredAttrDef("city", "string")); + /******* Organization Level Type *******/ + AtlasEnumDef orgLevelType = new AtlasEnumDef(ORG_LEVEL_TYPE, description(ORG_LEVEL_TYPE), DEFAULT_VERSION, + getOrgLevelElements()); + + /******* Security Clearance Type *******/ + AtlasClassificationDef securityClearanceType = createTraitTypeDef(SECURITY_CLEARANCE_TYPE, description(SECURITY_CLEARANCE_TYPE), + superType(null), createRequiredAttrDef("level", "int")); + + /******* [Department -> Employee] Relationship *******/ + AtlasRelationshipDef employeeDepartmentType = new AtlasRelationshipDef(EMPLOYEE_DEPARTMENT_TYPE, description(EMPLOYEE_DEPARTMENT_TYPE), + DEFAULT_VERSION, AGGREGATION, ONE_TO_TWO, + new AtlasRelationshipEndDef(EMPLOYEE_TYPE, "department", SINGLE), + new AtlasRelationshipEndDef(DEPARTMENT_TYPE, "employees", SET, true)); + /******* [Manager -> Employee] Relationship *******/ + AtlasRelationshipDef employeeManagerType = new AtlasRelationshipDef(EMPLOYEE_MANAGER_TYPE, description(EMPLOYEE_MANAGER_TYPE), + DEFAULT_VERSION, AGGREGATION, ONE_TO_TWO, + new AtlasRelationshipEndDef(EMPLOYEE_TYPE, "manager", SINGLE), + new AtlasRelationshipEndDef(MANAGER_TYPE, "subordinates", SET, true)); + + /******* [Mentor -> Employee] Relationship *******/ + AtlasRelationshipDef employeeMentorType = new AtlasRelationshipDef(EMPLOYEE_MENTOR_TYPE, description(EMPLOYEE_MENTOR_TYPE), + DEFAULT_VERSION, ASSOCIATION, ONE_TO_TWO, + new AtlasRelationshipEndDef(EMPLOYEE_TYPE, "mentor", SINGLE), + new AtlasRelationshipEndDef(EMPLOYEE_TYPE, "mentees", SET)); + + return new AtlasTypesDef(ImmutableList.of(orgLevelType), + ImmutableList.of(addressType), + ImmutableList.of(securityClearanceType), + ImmutableList.of(personType, employeeType, departmentType, managerType), + ImmutableList.of(employeeDepartmentType, employeeManagerType, employeeMentorType)); + } + + public static AtlasEntitiesWithExtInfo getDepartmentEmployeeInstances() { + AtlasEntitiesWithExtInfo ret = new AtlasEntitiesWithExtInfo(); + + /******* Department - HR *******/ + AtlasEntity hrDept = new AtlasEntity(DEPARTMENT_TYPE, "name", "hr"); + + /******* Address *******/ + AtlasStruct janeAddr = new AtlasStruct(ADDRESS_TYPE); + janeAddr.setAttribute("street", "Great America Parkway"); + janeAddr.setAttribute("city", "Santa Clara"); + + AtlasStruct juliusAddr = new AtlasStruct(ADDRESS_TYPE); + juliusAddr.setAttribute("street", "Madison Ave"); + juliusAddr.setAttribute("city", "Newtonville"); + + AtlasStruct maxAddr = new AtlasStruct(ADDRESS_TYPE); + maxAddr.setAttribute("street", "Ripley St"); + maxAddr.setAttribute("city", "Newton"); + + AtlasStruct johnAddr = new AtlasStruct(ADDRESS_TYPE); + johnAddr.setAttribute("street", "Stewart Drive"); + johnAddr.setAttribute("city", "Sunnyvale"); + + /******* Manager - Jane (John and Max subordinates) *******/ + AtlasEntity jane = new AtlasEntity(MANAGER_TYPE); + jane.setAttribute("name", "Jane"); + jane.setRelationshipAttribute("department", getAtlasObjectId(hrDept)); + jane.setAttribute("address", janeAddr); + + /******* Manager - Julius (no subordinates) *******/ + AtlasEntity julius = new AtlasEntity(MANAGER_TYPE); + julius.setAttribute("name", "Julius"); + julius.setRelationshipAttribute("department", getAtlasObjectId(hrDept)); + julius.setAttribute("address", juliusAddr); + + /******* Employee - Max (Manager: Jane, Mentor: Julius) *******/ + AtlasEntity max = new AtlasEntity(EMPLOYEE_TYPE); + max.setAttribute("name", "Max"); + max.setRelationshipAttribute("department", getAtlasObjectId(hrDept)); + max.setAttribute("address", maxAddr); + max.setRelationshipAttribute("manager", getAtlasObjectId(jane)); + max.setRelationshipAttribute("mentor", getAtlasObjectId(julius)); + max.setAttribute("birthday",new Date(1979, 3, 15)); + max.setAttribute("hasPets", true); + max.setAttribute("age", 36); + max.setAttribute("numberOfCars", 2); + max.setAttribute("houseNumber", 17); + max.setAttribute("carMileage", 13); + max.setAttribute("shares", Long.MAX_VALUE); + max.setAttribute("salary", Double.MAX_VALUE); + max.setAttribute("numberOfStarsEstimate", new BigInteger("1000000000000000000000000000000")); + max.setAttribute("approximationOfPi", new BigDecimal("3.1415926535897932")); + + /******* Employee - John (Manager: Jane, Mentor: Max) *******/ + AtlasEntity john = new AtlasEntity(EMPLOYEE_TYPE); + john.setAttribute("name", "John"); + john.setRelationshipAttribute("department", getAtlasObjectId(hrDept)); + john.setAttribute("address", johnAddr); + john.setRelationshipAttribute("manager", getAtlasObjectId(jane)); + john.setRelationshipAttribute("mentor", getAtlasObjectId(max)); + john.setAttribute("birthday",new Date(1950, 5, 15)); + john.setAttribute("hasPets", true); + john.setAttribute("numberOfCars", 1); + john.setAttribute("houseNumber", 153); + john.setAttribute("carMileage", 13364); + john.setAttribute("shares", 15000); + john.setAttribute("salary", 123345.678); + john.setAttribute("age", 50); + john.setAttribute("numberOfStarsEstimate", new BigInteger("1000000000000000000000")); + john.setAttribute("approximationOfPi", new BigDecimal("3.141592653589793238462643383279502884197169399375105820974944592307816406286")); + + ret.addEntity(hrDept); + ret.addEntity(jane); + ret.addEntity(julius); + ret.addEntity(max); + ret.addEntity(john); + + return ret; + } + + public static AtlasTypesDef getInverseReferenceTestTypes() throws AtlasBaseException { + AtlasEntityDef aType = createClassTypeDef(TYPE_A, superType(null), createUniqueRequiredAttrDef("name", "string")); + AtlasEntityDef bType = createClassTypeDef(TYPE_B, superType(null), createUniqueRequiredAttrDef("name", "string")); + + AtlasRelationshipDef relationshipType1 = new AtlasRelationshipDef("TypeA_to_TypeB_on_b", description("TypeA_to_TypeB_on_b"), + DEFAULT_VERSION, ASSOCIATION, ONE_TO_TWO, + new AtlasRelationshipEndDef(TYPE_A, "b", SINGLE), + new AtlasRelationshipEndDef(TYPE_B, "a", SINGLE)); + + AtlasRelationshipDef relationshipType2 = new AtlasRelationshipDef("TypeA_to_TypeB_on_oneB", description("TypeA_to_TypeB_on_oneB"), + DEFAULT_VERSION, ASSOCIATION, ONE_TO_TWO, + new AtlasRelationshipEndDef(TYPE_A, "oneB", SINGLE), + new AtlasRelationshipEndDef(TYPE_B, "manyA", SET)); + + AtlasRelationshipDef relationshipType3 = new AtlasRelationshipDef("TypeA_to_TypeB_on_manyB", description("TypeA_to_TypeB_on_manyB"), + DEFAULT_VERSION, ASSOCIATION, ONE_TO_TWO, + new AtlasRelationshipEndDef(TYPE_A, "manyB", SET), + new AtlasRelationshipEndDef(TYPE_B, "manyToManyA", SET)); + + AtlasRelationshipDef relationshipType4 = new AtlasRelationshipDef("TypeB_to_TypeA_on_mappedFromA", description("TypeB_to_TypeA_on_mappedFromA"), + DEFAULT_VERSION, ASSOCIATION, ONE_TO_TWO, + new AtlasRelationshipEndDef(TYPE_B, "mappedFromA", SINGLE), + new AtlasRelationshipEndDef(TYPE_A, "mapToB", SET)); + + return new AtlasTypesDef(ImmutableList.<AtlasEnumDef>of(), ImmutableList.<AtlasStructDef>of(), + ImmutableList.<AtlasClassificationDef>of(), ImmutableList.of(aType, bType), + ImmutableList.of(relationshipType1, relationshipType2, relationshipType3, relationshipType4)); + } + + private static List<AtlasEnumElementDef> getOrgLevelElements() { + return Arrays.asList( + new AtlasEnumElementDef("L1", description("L1"), 1), + new AtlasEnumElementDef("L2", description("L2"), 2), + new AtlasEnumElementDef("L3", description("L3"), 3) + ); + } + + private static String description(String typeName) { + return typeName + " description"; + } + + private static ImmutableSet<String> superType(String superTypeName) { + return StringUtils.isNotEmpty(superTypeName) ? ImmutableSet.of(superTypeName) : ImmutableSet.<String>of(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java index 9774583..14614f1 100755 --- a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java +++ b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java @@ -35,7 +35,6 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef; import org.apache.atlas.model.typedef.AtlasTypesDef; -import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.type.AtlasTypeUtil; import org.apache.commons.lang.RandomStringUtils; http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index 6f6d74b..c47a89e 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -26,8 +26,10 @@ import org.apache.atlas.AtlasException; import org.apache.atlas.RequestContext; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntity.Status; +import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasRelationship; import org.apache.atlas.model.typedef.AtlasRelationshipDef; +import org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection; import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.RepositoryException; import org.apache.atlas.repository.graphdb.AtlasEdge; @@ -39,6 +41,7 @@ import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasRelationshipType; +import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.type.AtlasType; import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.atlas.typesystem.ITypedInstance; @@ -344,12 +347,46 @@ public final class GraphHelper { return null; } + public Iterator<AtlasEdge> getIncomingEdgesByLabel(AtlasVertex instanceVertex, String edgeLabel) { + return getAdjacentEdgesByLabel(instanceVertex, AtlasEdgeDirection.IN, edgeLabel); + } + public Iterator<AtlasEdge> getOutGoingEdgesByLabel(AtlasVertex instanceVertex, String edgeLabel) { return getAdjacentEdgesByLabel(instanceVertex, AtlasEdgeDirection.OUT, edgeLabel); } - public Iterator<AtlasEdge> getBothEdgesByLabel(AtlasVertex instanceVertex, String edgeLabel) { - return getAdjacentEdgesByLabel(instanceVertex, AtlasEdgeDirection.BOTH, edgeLabel); + public AtlasEdge getEdgeForLabel(AtlasVertex vertex, String edgeLabel, AtlasRelationshipEdgeDirection edgeDirection) { + AtlasEdge ret; + + switch (edgeDirection) { + case IN: + ret = getEdgeForLabel(vertex, edgeLabel, AtlasEdgeDirection.IN); + break; + + case OUT: + default: + ret = getEdgeForLabel(vertex, edgeLabel, AtlasEdgeDirection.OUT); + break; + } + + return ret; + } + + public Iterator<AtlasEdge> getEdgesForLabel(AtlasVertex vertex, String edgeLabel, AtlasRelationshipEdgeDirection edgeDirection) { + Iterator<AtlasEdge> ret; + + switch (edgeDirection) { + case IN: + ret = getIncomingEdgesByLabel(vertex, edgeLabel); + break; + + case OUT: + default: + ret = getOutGoingEdgesByLabel(vertex, edgeLabel); + break; + } + + return ret; } /** @@ -360,7 +397,11 @@ public final class GraphHelper { * @return */ public AtlasEdge getEdgeForLabel(AtlasVertex vertex, String edgeLabel) { - Iterator<AtlasEdge> iterator = getAdjacentEdgesByLabel(vertex, AtlasEdgeDirection.OUT, edgeLabel); + return getEdgeForLabel(vertex, edgeLabel, AtlasEdgeDirection.OUT); + } + + public AtlasEdge getEdgeForLabel(AtlasVertex vertex, String edgeLabel, AtlasEdgeDirection edgeDirection) { + Iterator<AtlasEdge> iterator = getAdjacentEdgesByLabel(vertex, edgeDirection, edgeLabel); AtlasEdge latestDeletedEdge = null; long latestDeletedEdgeTime = Long.MIN_VALUE; @@ -1280,4 +1321,43 @@ public final class GraphHelper { return ret; } + + public static boolean isRelationshipEdge(AtlasEdge edge) { + if (edge == null) { + return false; + } + + String edgeLabel = edge.getLabel(); + + return StringUtils.isNotEmpty(edge.getLabel()) ? edgeLabel.startsWith("r:") : false; + } + + public static AtlasObjectId getReferenceObjectId(AtlasEdge edge, AtlasRelationshipEdgeDirection relationshipDirection) { + AtlasObjectId ret = null; + + if (relationshipDirection == AtlasRelationshipEdgeDirection.OUT) { + ret = new AtlasObjectId(getGuid(edge.getInVertex()), getTypeName(edge.getInVertex())); + + } else if (relationshipDirection == AtlasRelationshipEdgeDirection.IN) { + ret = new AtlasObjectId(getGuid(edge.getOutVertex()), getTypeName(edge.getOutVertex())); + } + + return ret; + } + + public static AtlasObjectId getCurrentObjectId(AtlasEdge edge, AtlasRelationshipEdgeDirection relationshipDirection) { + String typeName = null; + String guid = null; + + if (relationshipDirection == AtlasRelationshipEdgeDirection.OUT) { + typeName = GraphHelper.getTypeName(edge.getOutVertex()); + guid = GraphHelper.getGuid(edge.getOutVertex()); + + } else if (relationshipDirection == AtlasRelationshipEdgeDirection.IN) { + typeName = GraphHelper.getTypeName(edge.getInVertex()); + guid = GraphHelper.getGuid(edge.getInVertex()); + } + + return new AtlasObjectId(guid, typeName); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java index 12e8bb1..0210a11 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java @@ -286,7 +286,7 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery { return; } - AtlasStruct struct; + final AtlasStruct struct; if (val instanceof AtlasStruct) { struct = (AtlasStruct) val; @@ -298,6 +298,10 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery { throw new AtlasBaseException(AtlasErrorCode.INVALID_STRUCT_VALUE, val.toString()); } + visitStruct(structType, struct); + } + + void visitStruct(AtlasStructType structType, AtlasStruct struct) throws AtlasBaseException { for (AtlasAttribute attribute : structType.getAllAttributes().values()) { AtlasType attrType = attribute.getAttributeType(); Object attrVal = struct.getAttribute(attribute.getName()); @@ -306,6 +310,16 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery { } } + void visitEntity(AtlasEntityType entityType, AtlasEntity entity) throws AtlasBaseException { + visitStruct(entityType, entity); + + for (AtlasAttribute attribute : entityType.getRelationshipAttributes().values()) { + AtlasType attrType = attribute.getAttributeType(); + Object attrVal = entity.getRelationshipAttribute(attribute.getName()); + + visitAttribute(attrType, attrVal); + } + } void walkEntityGraph(AtlasEntity entity) throws AtlasBaseException { if (entity == null) { @@ -316,7 +330,7 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery { recordObjectReference(entity.getGuid()); - visitStruct(type, entity); + visitEntity(type, entity); } http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java index 43f2c55..948d9dd 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java @@ -94,8 +94,8 @@ public class AtlasGraphUtilsV1 { public static String getQualifiedAttributePropertyKey(AtlasStructType fromType, String attributeName) throws AtlasBaseException { switch (fromType.getTypeCategory()) { - case STRUCT: case ENTITY: + case STRUCT: case CLASSIFICATION: return fromType.getQualifiedAttributeName(attributeName); default: http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java index 3ff6fbe..49e08a0 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java @@ -29,9 +29,11 @@ import org.apache.atlas.repository.RepositoryException; import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.graphdb.GremlinVersion; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasRelationshipType; +import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.type.AtlasStructType.AtlasAttribute; import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; @@ -78,35 +80,7 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore { AtlasVertex end1Vertex = getVertexFromEndPoint(relationship.getEnd1()); AtlasVertex end2Vertex = getVertexFromEndPoint(relationship.getEnd2()); - AtlasRelationship ret; - - // create relationship between two vertex - try { - AtlasEdge relationshipEdge = getRelationshipEdge(end1Vertex, end2Vertex, relationship); - - if (relationshipEdge == null) { - relationshipEdge = createRelationshipEdge(end1Vertex, end2Vertex, relationship); - - AtlasRelationshipType relationType = typeRegistry.getRelationshipTypeByName(relationship.getTypeName()); - - if (MapUtils.isNotEmpty(relationType.getAllAttributes())) { - for (AtlasAttribute attr : relationType.getAllAttributes().values()) { - String attrName = attr.getName(); - Object attrValue = relationship.getAttribute(attrName); - - AtlasGraphUtilsV1.setProperty(relationshipEdge, attr.getVertexPropertyName(), attrValue); - } - } - - ret = mapEdgeToAtlasRelationship(relationshipEdge); - - } else { - throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIP_ALREADY_EXISTS, relationship.getTypeName(), - relationship.getEnd1().getGuid(), relationship.getEnd2().getGuid()); - } - } catch (RepositoryException e) { - throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, e); - } + AtlasRelationship ret = createRelationship(relationship, end1Vertex, end2Vertex); if (LOG.isDebugEnabled()) { LOG.debug("<== create({}): {}", relationship, ret); @@ -117,31 +91,6 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore { @Override @GraphTransaction - public AtlasRelationship getOrCreate(AtlasRelationship relationship) throws AtlasBaseException { - if (LOG.isDebugEnabled()) { - LOG.debug("==> getOrCreate({})", relationship); - } - - validateRelationship(relationship); - - AtlasVertex end1Vertex = getVertexFromEndPoint(relationship.getEnd1()); - AtlasVertex end2Vertex = getVertexFromEndPoint(relationship.getEnd2()); - AtlasRelationship ret; - - // check if relationship exists - AtlasEdge relationshipEdge = getRelationshipEdge(end1Vertex, end2Vertex, relationship); - - ret = (relationshipEdge != null) ? mapEdgeToAtlasRelationship(relationshipEdge) : create(relationship); - - if (LOG.isDebugEnabled()) { - LOG.debug("<== getOrCreate({}): {}", relationship, ret); - } - - return ret; - } - - @Override - @GraphTransaction public AtlasRelationship update(AtlasRelationship relationship) throws AtlasBaseException { if (LOG.isDebugEnabled()) { LOG.debug("==> update({})", relationship); @@ -196,6 +145,70 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore { } } + public AtlasRelationship getOrCreate(AtlasRelationship relationship) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> getOrCreate({})", relationship); + } + + validateRelationship(relationship); + + AtlasVertex end1Vertex = getVertexFromEndPoint(relationship.getEnd1()); + AtlasVertex end2Vertex = getVertexFromEndPoint(relationship.getEnd2()); + AtlasRelationship ret; + + // check if relationship exists + AtlasEdge relationshipEdge = getRelationshipEdge(end1Vertex, end2Vertex, relationship); + + if (relationshipEdge != null) { + ret = mapEdgeToAtlasRelationship(relationshipEdge); + + } else { + validateRelationship(relationship); + ret = createRelationship(relationship, end1Vertex, end2Vertex); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== getOrCreate({}): {}", relationship, ret); + } + + return ret; + } + + private AtlasRelationship createRelationship(AtlasRelationship relationship, AtlasVertex end1Vertex, AtlasVertex end2Vertex) + throws AtlasBaseException { + AtlasRelationship ret; + + try { + AtlasEdge relationshipEdge = getRelationshipEdge(end1Vertex, end2Vertex, relationship); + + if (relationshipEdge == null) { + relationshipEdge = createRelationshipEdge(end1Vertex, end2Vertex, relationship); + + AtlasRelationshipType relationType = typeRegistry.getRelationshipTypeByName(relationship.getTypeName()); + + if (MapUtils.isNotEmpty(relationType.getAllAttributes())) { + for (AtlasAttribute attr : relationType.getAllAttributes().values()) { + String attrName = attr.getName(); + String attrVertexProperty = attr.getVertexPropertyName(); + Object attrValue = relationship.getAttribute(attrName); + + AtlasGraphUtilsV1.setProperty(relationshipEdge, attrVertexProperty, attrValue); + } + } + + ret = mapEdgeToAtlasRelationship(relationshipEdge); + + } else { + throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIP_ALREADY_EXISTS, relationship.getTypeName(), + relationship.getEnd1().getGuid(), relationship.getEnd2().getGuid()); + } + } catch (RepositoryException e) { + throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, e); + } + + return ret; + } + private void validateRelationship(AtlasRelationship relationship) throws AtlasBaseException { if (relationship == null) { throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "AtlasRelationship is null"); @@ -229,6 +242,7 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore { } validateEnd(relationship.getEnd1()); + validateEnd(relationship.getEnd2()); validateAndNormalize(relationship); @@ -273,7 +287,7 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore { type.getNormalizedValue(relationship); } - private AtlasEdge getRelationshipEdge(AtlasVertex fromVertex, AtlasVertex toVertex, AtlasRelationship relationship) { + public AtlasEdge getRelationshipEdge(AtlasVertex fromVertex, AtlasVertex toVertex, AtlasRelationship relationship) { String relationshipLabel = getRelationshipEdgeLabel(fromVertex, toVertex, relationship); AtlasEdge ret = graphHelper.getEdgeForLabel(fromVertex, relationshipLabel); @@ -331,31 +345,29 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore { } private String getRelationshipEdgeLabel(AtlasVertex fromVertex, AtlasVertex toVertex, AtlasRelationship relationship) { - - String ret = relationship.getRelationshipLabel(); AtlasRelationshipType relationshipType = typeRegistry.getRelationshipTypeByName(relationship.getTypeName()); + String ret = relationshipType.getRelationshipDef().getRelationshipLabel(); AtlasRelationshipEndDef endDef1 = relationshipType.getRelationshipDef().getEndDef1(); AtlasRelationshipEndDef endDef2 = relationshipType.getRelationshipDef().getEndDef2(); Set<String> fromVertexTypes = getTypeAndAllSuperTypes(AtlasGraphUtilsV1.getTypeName(fromVertex)); Set<String> toVertexTypes = getTypeAndAllSuperTypes(AtlasGraphUtilsV1.getTypeName(toVertex)); + AtlasAttribute attribute = null; // validate entity type and all its supertypes contains relationshipDefs end type - // e.g. [ hive_process -> hive_table] -> [ Process -> DataSet ] + // e.g. [hive_process -> hive_table] -> [Process -> DataSet] if (fromVertexTypes.contains(endDef1.getType()) && toVertexTypes.contains(endDef2.getType())) { - String attributeName = endDef1.getName(); - AtlasAttribute endAttribute = relationshipType.getEnd1Type().getAttribute(attributeName); + String attributeName = endDef1.getName(); - if (endAttribute != null) { - ret = endAttribute.getRelationshipEdgeLabel(); - } + attribute = relationshipType.getEnd1Type().getRelationshipAttribute(attributeName); } else if (fromVertexTypes.contains(endDef2.getType()) && toVertexTypes.contains(endDef1.getType())) { - String attributeName = endDef2.getName(); - AtlasAttribute endAttribute = relationshipType.getEnd2Type().getAttribute(attributeName); + String attributeName = endDef2.getName(); - if (endAttribute != null) { - ret = endAttribute.getRelationshipEdgeLabel(); - } + attribute = relationshipType.getEnd2Type().getRelationshipAttribute(attributeName); + } + + if (attribute != null) { + ret = attribute.getRelationshipEdgeLabel(); } return ret; @@ -391,7 +403,13 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore { relationship.setCreateTime(new Date(GraphHelper.getCreatedTime(edge))); relationship.setUpdateTime(new Date(GraphHelper.getModifiedTime(edge))); - relationship.setVersion(GraphHelper.getVersion(edge).longValue()); + Integer version = GraphHelper.getVersion(edge); + + if (version == null) { + version = Integer.valueOf(1); + } + + relationship.setVersion(version.longValue()); relationship.setStatus(GraphHelper.getEdgeStatus(edge)); AtlasVertex end1Vertex = edge.getOutVertex(); http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java index d4fdc25..4271376 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java @@ -37,6 +37,7 @@ import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasMapType; import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.type.AtlasStructType.AtlasAttribute; +import org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection; import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; import org.slf4j.Logger; @@ -53,6 +54,8 @@ import java.util.Set; import java.util.Stack; import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX; +import static org.apache.atlas.repository.graph.GraphHelper.getReferenceObjectId; +import static org.apache.atlas.repository.graph.GraphHelper.isRelationshipEdge; import static org.apache.atlas.repository.graph.GraphHelper.string; public abstract class DeleteHandlerV1 { @@ -206,7 +209,6 @@ public abstract class DeleteHandlerV1 { return result; } - /** * Force delete is used to remove struct/trait in case of entity updates * @param edge @@ -217,13 +219,20 @@ public abstract class DeleteHandlerV1 { * @throws AtlasException */ public boolean deleteEdgeReference(AtlasEdge edge, TypeCategory typeCategory, boolean isOwned, - boolean forceDeleteStructTrait) throws AtlasBaseException { + boolean forceDeleteStructTrait) throws AtlasBaseException { + + // default edge direction is outward + return deleteEdgeReference(edge, typeCategory, isOwned, forceDeleteStructTrait, AtlasRelationshipEdgeDirection.OUT); + } + + public boolean deleteEdgeReference(AtlasEdge edge, TypeCategory typeCategory, boolean isOwned, boolean forceDeleteStructTrait, + AtlasRelationshipEdgeDirection relationshipDirection) throws AtlasBaseException { LOG.debug("Deleting {}", string(edge)); boolean forceDelete = - (typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION) && forceDeleteStructTrait; + (typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION) && forceDeleteStructTrait; if (typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION - || (typeCategory == TypeCategory.OBJECT_ID_TYPE && isOwned)) { + || (typeCategory == TypeCategory.OBJECT_ID_TYPE && isOwned)) { //If the vertex is of type struct/trait, delete the edge and then the reference vertex as the vertex is not shared by any other entities. //If the vertex is of type class, and its composite attribute, this reference vertex' lifecycle is controlled //through this delete, hence delete the edge and the reference vertex. @@ -236,9 +245,19 @@ public abstract class DeleteHandlerV1 { //If the vertex is of type class, and its not a composite attributes, the reference AtlasVertex' lifecycle is not controlled //through this delete. Hence just remove the reference edge. Leave the reference AtlasVertex as is - //If deleting just the edge, reverse attribute should be updated for any references - //For example, for the department type system, if the person's manager edge is deleted, subordinates of manager should be updated - deleteEdge(edge, true, false); + // for relationship edges, inverse vertex's relationship attribute doesn't need to be updated. + // only delete the reference relationship edge + if (isRelationshipEdge(edge)) { + deleteEdge(edge, false); + + AtlasObjectId deletedReferenceObjectId = getReferenceObjectId(edge, relationshipDirection); + RequestContextV1.get().recordEntityUpdate(deletedReferenceObjectId); + } else { + //legacy case - not a relationship edge + //If deleting just the edge, reverse attribute should be updated for any references + //For example, for the department type system, if the person's manager edge is deleted, subordinates of manager should be updated + deleteEdge(edge, true, false); + } } return !softDelete || forceDelete; }
