Repository: atlas Updated Branches: refs/heads/master 12622e020 -> d1aa36c46
ATLAS-2283: add subTypes field in AtlasClassificationDef and AtlasEntityDef Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/1e75da0d Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/1e75da0d Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/1e75da0d Branch: refs/heads/master Commit: 1e75da0d2787b032390360acf05fdbb5af97e2fc Parents: 12622e0 Author: Madhan Neethiraj <[email protected]> Authored: Mon Nov 20 03:18:01 2017 -0800 Committer: Madhan Neethiraj <[email protected]> Committed: Fri Dec 1 08:21:11 2017 -0800 ---------------------------------------------------------------------- .../model/typedef/AtlasClassificationDef.java | 12 +++ .../atlas/model/typedef/AtlasEntityDef.java | 12 +++ .../atlas/type/AtlasClassificationType.java | 17 +++- .../org/apache/atlas/type/AtlasEntityType.java | 20 ++++- .../atlas/type/TestAtlasTypeRegistry.java | 91 ++++++++++++++------ 5 files changed, 122 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/1e75da0d/intg/src/main/java/org/apache/atlas/model/typedef/AtlasClassificationDef.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasClassificationDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasClassificationDef.java index 2a9e08c..f6a6927 100644 --- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasClassificationDef.java +++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasClassificationDef.java @@ -55,6 +55,10 @@ public class AtlasClassificationDef extends AtlasStructDef implements java.io.Se private Set<String> superTypes; private Set<String> entityTypes; + // subTypes field below is derived from 'superTypes' specified in all AtlasClassificationDef + // this value is ignored during create & update operations + private Set<String> subTypes; + public AtlasClassificationDef() { this(null, null, null, null, null, null); @@ -119,6 +123,14 @@ public class AtlasClassificationDef extends AtlasStructDef implements java.io.Se } } + public Set<String> getSubTypes() { + return subTypes; + } + + public void setSubTypes(Set<String> subTypes) { + this.subTypes = subTypes; + } + public boolean hasSuperType(String typeName) { return hasSuperType(superTypes, typeName); } http://git-wip-us.apache.org/repos/asf/atlas/blob/1e75da0d/intg/src/main/java/org/apache/atlas/model/typedef/AtlasEntityDef.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasEntityDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasEntityDef.java index 230fca1..36bb3df 100644 --- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasEntityDef.java +++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasEntityDef.java @@ -54,6 +54,10 @@ public class AtlasEntityDef extends AtlasStructDef implements java.io.Serializab private Set<String> superTypes; + // subTypes field below is derived from 'superTypes' specified in all AtlasEntityDef + // this value is ignored during create & update operations + private Set<String> subTypes; + public AtlasEntityDef() { this(null, null, null, null, null, null); @@ -109,6 +113,14 @@ public class AtlasEntityDef extends AtlasStructDef implements java.io.Serializab } } + public Set<String> getSubTypes() { + return subTypes; + } + + public void setSubTypes(Set<String> subTypes) { + this.subTypes = subTypes; + } + public boolean hasSuperType(String typeName) { return hasSuperType(superTypes, typeName); } http://git-wip-us.apache.org/repos/asf/atlas/blob/1e75da0d/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java index 9f39423..8413ce6 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java @@ -41,6 +41,7 @@ public class AtlasClassificationType extends AtlasStructType { private List<AtlasClassificationType> superTypes = Collections.emptyList(); private Set<String> allSuperTypes = Collections.emptySet(); + private Set<String> subTypes = Collections.emptySet(); private Set<String> allSubTypes = Collections.emptySet(); private Set<String> typeAndAllSubTypes = Collections.emptySet(); private String typeAndAllSubTypesQryStr = ""; @@ -103,6 +104,7 @@ public class AtlasClassificationType extends AtlasStructType { this.allSuperTypes = Collections.unmodifiableSet(allS); this.allAttributes = Collections.unmodifiableMap(allA); this.uniqAttributes = getUniqueAttributes(this.allAttributes); + this.subTypes = new HashSet<>(); // this will be populated in resolveReferencesPhase2() this.allSubTypes = new HashSet<>(); // this will be populated in resolveReferencesPhase2() this.typeAndAllSubTypes = new HashSet<>(); // this will be populated in resolveReferencesPhase2() this.entityTypes = new HashSet<>(); // this will be populated in resolveReferencesPhase3() @@ -114,9 +116,13 @@ public class AtlasClassificationType extends AtlasStructType { void resolveReferencesPhase2(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { super.resolveReferencesPhase2(typeRegistry); + for (AtlasClassificationType superType : superTypes) { + superType.addSubType(this); + } + for (String superTypeName : allSuperTypes) { AtlasClassificationType superType = typeRegistry.getClassificationTypeByName(superTypeName); - superType.addSubType(this); + superType.addToAllSubTypes(this); } } @@ -139,6 +145,7 @@ public class AtlasClassificationType extends AtlasStructType { */ @Override void resolveReferencesPhase3(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { + subTypes = Collections.unmodifiableSet(subTypes); allSubTypes = Collections.unmodifiableSet(allSubTypes); typeAndAllSubTypes = Collections.unmodifiableSet(typeAndAllSubTypes); typeAndAllSubTypesQryStr = ""; // will be computed on next access @@ -206,9 +213,15 @@ public class AtlasClassificationType extends AtlasStructType { } } } + + classificationDef.setSubTypes(subTypes); } private void addSubType(AtlasClassificationType subType) { + subTypes.add(subType.getTypeName()); + } + + private void addToAllSubTypes(AtlasClassificationType subType) { allSubTypes.add(subType.getTypeName()); typeAndAllSubTypes.add(subType.getTypeName()); } @@ -219,6 +232,8 @@ public class AtlasClassificationType extends AtlasStructType { public Set<String> getAllSuperTypes() { return allSuperTypes; } + public Set<String> getSubTypes() { return subTypes; } + public Set<String> getAllSubTypes() { return allSubTypes; } public Set<String> getTypeAndAllSubTypes() { return typeAndAllSubTypes; } http://git-wip-us.apache.org/repos/asf/atlas/blob/1e75da0d/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java index 205e5b9..4aeaf47 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java @@ -51,6 +51,7 @@ public class AtlasEntityType extends AtlasStructType { private List<AtlasEntityType> superTypes = Collections.emptyList(); private Set<String> allSuperTypes = Collections.emptySet(); + private Set<String> subTypes = Collections.emptySet(); private Set<String> allSubTypes = Collections.emptySet(); private Set<String> typeAndAllSubTypes = Collections.emptySet(); private Set<String> typeAndAllSuperTypes = Collections.emptySet(); @@ -58,6 +59,7 @@ public class AtlasEntityType extends AtlasStructType { private Map<String, List<AtlasRelationshipType>> relationshipAttributesType = Collections.emptyMap(); private String typeAndAllSubTypesQryStr = ""; + public AtlasEntityType(AtlasEntityDef entityDef) { super(entityDef); @@ -98,6 +100,7 @@ public class AtlasEntityType extends AtlasStructType { this.allSuperTypes = Collections.unmodifiableSet(allS); this.allAttributes = Collections.unmodifiableMap(allA); this.uniqAttributes = getUniqueAttributes(this.allAttributes); + this.subTypes = new HashSet<>(); // this will be populated in resolveReferencesPhase2() this.allSubTypes = new HashSet<>(); // this will be populated in resolveReferencesPhase2() this.typeAndAllSubTypes = new HashSet<>(); // this will be populated in resolveReferencesPhase2() this.relationshipAttributes = new HashMap<>(); // this will be populated in resolveReferencesPhase3() @@ -114,9 +117,13 @@ public class AtlasEntityType extends AtlasStructType { void resolveReferencesPhase2(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { super.resolveReferencesPhase2(typeRegistry); + for (AtlasEntityType superType : superTypes) { + superType.addSubType(this); + } + for (String superTypeName : allSuperTypes) { AtlasEntityType superType = typeRegistry.getEntityTypeByName(superTypeName); - superType.addSubType(this); + superType.addToAllSubTypes(this); } } @@ -150,11 +157,14 @@ public class AtlasEntityType extends AtlasStructType { } } + subTypes = Collections.unmodifiableSet(subTypes); allSubTypes = Collections.unmodifiableSet(allSubTypes); typeAndAllSubTypes = Collections.unmodifiableSet(typeAndAllSubTypes); typeAndAllSubTypesQryStr = ""; // will be computed on next access relationshipAttributes = Collections.unmodifiableMap(relationshipAttributes); relationshipAttributesType = Collections.unmodifiableMap(relationshipAttributesType); + + entityDef.setSubTypes(subTypes); } public Set<String> getSuperTypes() { @@ -165,6 +175,8 @@ public class AtlasEntityType extends AtlasStructType { return allSuperTypes; } + public Set<String> getSubTypes() { return subTypes; } + public Set<String> getAllSubTypes() { return allSubTypes; } public Set<String> getTypeAndAllSubTypes() { return typeAndAllSubTypes; } @@ -435,6 +447,10 @@ public class AtlasEntityType extends AtlasStructType { } private void addSubType(AtlasEntityType subType) { + subTypes.add(subType.getTypeName()); + } + + private void addToAllSubTypes(AtlasEntityType subType) { allSubTypes.add(subType.getTypeName()); typeAndAllSubTypes.add(subType.getTypeName()); } @@ -679,4 +695,4 @@ public class AtlasEntityType extends AtlasStructType { return ret; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/atlas/blob/1e75da0d/intg/src/test/java/org/apache/atlas/type/TestAtlasTypeRegistry.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/type/TestAtlasTypeRegistry.java b/intg/src/test/java/org/apache/atlas/type/TestAtlasTypeRegistry.java index accba77..476bc33 100644 --- a/intg/src/test/java/org/apache/atlas/type/TestAtlasTypeRegistry.java +++ b/intg/src/test/java/org/apache/atlas/type/TestAtlasTypeRegistry.java @@ -101,15 +101,15 @@ public class TestAtlasTypeRegistry { assertNull(failureMsg); - validateSuperTypes(typeRegistry, "L0", new HashSet<String>()); - validateSuperTypes(typeRegistry, "L1-1", new HashSet<>(Arrays.asList("L0"))); - validateSuperTypes(typeRegistry, "L1-2", new HashSet<>(Arrays.asList("L0"))); - validateSuperTypes(typeRegistry, "L2-1", new HashSet<>(Arrays.asList("L1-1", "L0"))); - validateSuperTypes(typeRegistry, "L2-2", new HashSet<>(Arrays.asList("L1-1", "L0"))); - validateSuperTypes(typeRegistry, "L2-3", new HashSet<>(Arrays.asList("L1-1", "L0", "L1-2"))); - validateSuperTypes(typeRegistry, "L2-4", new HashSet<>(Arrays.asList("L1-2", "L0"))); - - validateSubTypes(typeRegistry, "L0", new HashSet<>(Arrays.asList("L1-1", "L1-2", "L2-1", "L2-2", "L2-3", "L2-4"))); + validateAllSuperTypes(typeRegistry, "L0", new HashSet<String>()); + validateAllSuperTypes(typeRegistry, "L1-1", new HashSet<>(Arrays.asList("L0"))); + validateAllSuperTypes(typeRegistry, "L1-2", new HashSet<>(Arrays.asList("L0"))); + validateAllSuperTypes(typeRegistry, "L2-1", new HashSet<>(Arrays.asList("L1-1", "L0"))); + validateAllSuperTypes(typeRegistry, "L2-2", new HashSet<>(Arrays.asList("L1-1", "L0"))); + validateAllSuperTypes(typeRegistry, "L2-3", new HashSet<>(Arrays.asList("L1-1", "L0", "L1-2"))); + validateAllSuperTypes(typeRegistry, "L2-4", new HashSet<>(Arrays.asList("L1-2", "L0"))); + + validateSubTypes(typeRegistry, "L0", new HashSet<>(Arrays.asList("L1-1", "L1-2"))); validateSubTypes(typeRegistry, "L1-1", new HashSet<>(Arrays.asList("L2-1", "L2-2", "L2-3"))); validateSubTypes(typeRegistry, "L1-2", new HashSet<>(Arrays.asList("L2-3", "L2-4"))); validateSubTypes(typeRegistry, "L2-1", new HashSet<String>()); @@ -117,6 +117,14 @@ public class TestAtlasTypeRegistry { validateSubTypes(typeRegistry, "L2-3", new HashSet<String>()); validateSubTypes(typeRegistry, "L2-4", new HashSet<String>()); + validateAllSubTypes(typeRegistry, "L0", new HashSet<>(Arrays.asList("L1-1", "L1-2", "L2-1", "L2-2", "L2-3", "L2-4"))); + validateAllSubTypes(typeRegistry, "L1-1", new HashSet<>(Arrays.asList("L2-1", "L2-2", "L2-3"))); + validateAllSubTypes(typeRegistry, "L1-2", new HashSet<>(Arrays.asList("L2-3", "L2-4"))); + validateAllSubTypes(typeRegistry, "L2-1", new HashSet<String>()); + validateAllSubTypes(typeRegistry, "L2-2", new HashSet<String>()); + validateAllSubTypes(typeRegistry, "L2-3", new HashSet<String>()); + validateAllSubTypes(typeRegistry, "L2-4", new HashSet<String>()); + validateAttributeNames(typeRegistry, "L0", new HashSet<>(Arrays.asList("L0_a1"))); validateAttributeNames(typeRegistry, "L1-1", new HashSet<>(Arrays.asList("L0_a1", "L1-1_a1"))); validateAttributeNames(typeRegistry, "L1-2", new HashSet<>(Arrays.asList("L0_a1", "L1-2_a1"))); @@ -273,15 +281,15 @@ public class TestAtlasTypeRegistry { } assertNull(failureMsg); - validateSuperTypes(typeRegistry, "L0", new HashSet<String>()); - validateSuperTypes(typeRegistry, "L1-1", new HashSet<>(Arrays.asList("L0"))); - validateSuperTypes(typeRegistry, "L1-2", new HashSet<>(Arrays.asList("L0"))); - validateSuperTypes(typeRegistry, "L2-1", new HashSet<>(Arrays.asList("L1-1", "L0"))); - validateSuperTypes(typeRegistry, "L2-2", new HashSet<>(Arrays.asList("L1-1", "L0"))); - validateSuperTypes(typeRegistry, "L2-3", new HashSet<>(Arrays.asList("L1-1", "L0", "L1-2"))); - validateSuperTypes(typeRegistry, "L2-4", new HashSet<>(Arrays.asList("L1-2", "L0"))); + validateAllSuperTypes(typeRegistry, "L0", new HashSet<String>()); + validateAllSuperTypes(typeRegistry, "L1-1", new HashSet<>(Arrays.asList("L0"))); + validateAllSuperTypes(typeRegistry, "L1-2", new HashSet<>(Arrays.asList("L0"))); + validateAllSuperTypes(typeRegistry, "L2-1", new HashSet<>(Arrays.asList("L1-1", "L0"))); + validateAllSuperTypes(typeRegistry, "L2-2", new HashSet<>(Arrays.asList("L1-1", "L0"))); + validateAllSuperTypes(typeRegistry, "L2-3", new HashSet<>(Arrays.asList("L1-1", "L0", "L1-2"))); + validateAllSuperTypes(typeRegistry, "L2-4", new HashSet<>(Arrays.asList("L1-2", "L0"))); - validateSubTypes(typeRegistry, "L0", new HashSet<>(Arrays.asList("L1-1", "L1-2", "L2-1", "L2-2", "L2-3", "L2-4"))); + validateSubTypes(typeRegistry, "L0", new HashSet<>(Arrays.asList("L1-1", "L1-2"))); validateSubTypes(typeRegistry, "L1-1", new HashSet<>(Arrays.asList("L2-1", "L2-2", "L2-3"))); validateSubTypes(typeRegistry, "L1-2", new HashSet<>(Arrays.asList("L2-3", "L2-4"))); validateSubTypes(typeRegistry, "L2-1", new HashSet<String>()); @@ -289,6 +297,14 @@ public class TestAtlasTypeRegistry { validateSubTypes(typeRegistry, "L2-3", new HashSet<String>()); validateSubTypes(typeRegistry, "L2-4", new HashSet<String>()); + validateAllSubTypes(typeRegistry, "L0", new HashSet<>(Arrays.asList("L1-1", "L1-2", "L2-1", "L2-2", "L2-3", "L2-4"))); + validateAllSubTypes(typeRegistry, "L1-1", new HashSet<>(Arrays.asList("L2-1", "L2-2", "L2-3"))); + validateAllSubTypes(typeRegistry, "L1-2", new HashSet<>(Arrays.asList("L2-3", "L2-4"))); + validateAllSubTypes(typeRegistry, "L2-1", new HashSet<String>()); + validateAllSubTypes(typeRegistry, "L2-2", new HashSet<String>()); + validateAllSubTypes(typeRegistry, "L2-3", new HashSet<String>()); + validateAllSubTypes(typeRegistry, "L2-4", new HashSet<String>()); + validateAttributeNames(typeRegistry, "L0", new HashSet<>(Arrays.asList("L0_a1"))); validateAttributeNames(typeRegistry, "L1-1", new HashSet<>(Arrays.asList("L0_a1", "L1-1_a1"))); validateAttributeNames(typeRegistry, "L1-2", new HashSet<>(Arrays.asList("L0_a1", "L1-2_a1"))); @@ -519,11 +535,11 @@ public class TestAtlasTypeRegistry { } assertNull(failureMsg); - validateSuperTypes(typeRegistry, "L0", new HashSet<String>()); - validateSubTypes(typeRegistry, "L0", new HashSet<>(Arrays.asList("L1"))); + validateAllSuperTypes(typeRegistry, "L0", new HashSet<String>()); + validateAllSubTypes(typeRegistry, "L0", new HashSet<>(Arrays.asList("L1"))); - validateSuperTypes(typeRegistry, "L1", new HashSet<>(Arrays.asList("L0"))); - validateSubTypes(typeRegistry, "L1", new HashSet<String>()); + validateAllSuperTypes(typeRegistry, "L1", new HashSet<>(Arrays.asList("L0"))); + validateAllSubTypes(typeRegistry, "L1", new HashSet<String>()); // create a circular reference @@ -552,11 +568,11 @@ public class TestAtlasTypeRegistry { assertNull(typeRegistry.getEntityTypeByName("L2")); - validateSuperTypes(typeRegistry, "L0", new HashSet<String>()); - validateSubTypes(typeRegistry, "L0", new HashSet<>(Arrays.asList("L1"))); + validateAllSuperTypes(typeRegistry, "L0", new HashSet<String>()); + validateAllSubTypes(typeRegistry, "L0", new HashSet<>(Arrays.asList("L1"))); - validateSuperTypes(typeRegistry, "L1", new HashSet<>(Arrays.asList("L0"))); - validateSubTypes(typeRegistry, "L1", new HashSet<String>()); + validateAllSuperTypes(typeRegistry, "L1", new HashSet<>(Arrays.asList("L0"))); + validateAllSubTypes(typeRegistry, "L1", new HashSet<String>()); } private boolean addType(AtlasTypeRegistry typeRegistry, AtlasBaseTypeDef typeDef) { @@ -578,7 +594,7 @@ public class TestAtlasTypeRegistry { return ret; } - private void validateSuperTypes(AtlasTypeRegistry typeRegistry, String typeName, Set<String> expectedSuperTypes) { + private void validateAllSuperTypes(AtlasTypeRegistry typeRegistry, String typeName, Set<String> expectedSuperTypes) { AtlasType type = null; try { @@ -599,7 +615,7 @@ public class TestAtlasTypeRegistry { assertEquals(superTypes, expectedSuperTypes); } - private void validateSubTypes(AtlasTypeRegistry typeRegistry, String typeName, Set<String> expectedSubTypes) { + private void validateAllSubTypes(AtlasTypeRegistry typeRegistry, String typeName, Set<String> expectedSubTypes) { AtlasType type = null; try { @@ -620,6 +636,27 @@ public class TestAtlasTypeRegistry { assertEquals(subTypes, expectedSubTypes); } + private void validateSubTypes(AtlasTypeRegistry typeRegistry, String typeName, Set<String> expectedSubTypes) { + AtlasType type = null; + + try { + type = typeRegistry.getType(typeName); + } catch (AtlasBaseException excp) { + } + + Set<String> subTypes = null; + + if (type != null) { + if (type instanceof AtlasEntityType) { + subTypes = ((AtlasEntityType) type).getSubTypes(); + } else if (type instanceof AtlasClassificationType) { + subTypes = ((AtlasClassificationType) type).getSubTypes(); + } + } + + assertEquals(subTypes, expectedSubTypes); + } + private void validateAttributeNames(AtlasTypeRegistry typeRegistry, String typeName, Set<String> attributeNames) { AtlasType type = null;
