Repository: incubator-atlas Updated Branches: refs/heads/master 0c7468954 -> 6a24cad18
ATLAS-1268: added methods to retrieve attributeDefs in a type and all its superTypes Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/6a24cad1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/6a24cad1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/6a24cad1 Branch: refs/heads/master Commit: 6a24cad180b071290211c2695c567e83c13e9f97 Parents: 0c74689 Author: Madhan Neethiraj <[email protected]> Authored: Thu Nov 3 00:33:52 2016 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Thu Nov 3 13:17:54 2016 -0700 ---------------------------------------------------------------------- .../atlas/type/AtlasClassificationType.java | 60 +++++++---- .../org/apache/atlas/type/AtlasEntityType.java | 60 +++++++---- .../atlas/type/TestAtlasTypeRegistry.java | 105 ++++++++++++++++++- 3 files changed, 179 insertions(+), 46 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6a24cad1/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java index fa4ca20..d841977 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java @@ -22,12 +22,14 @@ import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasClassification; import org.apache.atlas.model.typedef.AtlasClassificationDef; +import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -42,8 +44,9 @@ public class AtlasClassificationType extends AtlasStructType { private final AtlasClassificationDef classificationDef; - private List<AtlasClassificationType> superTypes = Collections.emptyList(); - private Set<String> allSuperTypes = Collections.emptySet(); + private List<AtlasClassificationType> superTypes = Collections.emptyList(); + private Set<String> allSuperTypes = Collections.emptySet(); + private Map<String, AtlasAttributeDef> allAttributeDefs = Collections.emptyMap(); public AtlasClassificationType(AtlasClassificationDef classificationDef) { super(classificationDef, TypeCategory.CLASSIFICATION); @@ -64,8 +67,11 @@ public class AtlasClassificationType extends AtlasStructType { public void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { super.resolveReferences(typeRegistry); - List<AtlasClassificationType> s = new ArrayList<AtlasClassificationType>(); - Set<String> allS = getAllSuperTypes(typeRegistry); + List<AtlasClassificationType> s = new ArrayList<>(); + Set<String> allS = new HashSet<>(); + Map<String, AtlasAttributeDef> allA = new HashMap<>(); + + getTypeHierarchyInfo(typeRegistry, allS, allA); for (String superTypeName : classificationDef.getSuperTypes()) { AtlasType superType = typeRegistry.getType(superTypeName); @@ -78,8 +84,9 @@ public class AtlasClassificationType extends AtlasStructType { } } - this.superTypes = Collections.unmodifiableList(s); - this.allSuperTypes = Collections.unmodifiableSet(allS); + this.superTypes = Collections.unmodifiableList(s); + this.allSuperTypes = Collections.unmodifiableSet(allS); + this.allAttributeDefs = Collections.unmodifiableMap(allA); } public Set<String> getSuperTypes() { @@ -88,6 +95,8 @@ public class AtlasClassificationType extends AtlasStructType { public Set<String> getAllSuperTypes() { return allSuperTypes; } + public Map<String, AtlasAttributeDef> getAllAttributeDefs() { return allAttributeDefs; } + public boolean isSuperTypeOf(AtlasClassificationType classificationType) { return classificationType != null ? classificationType.getAllSuperTypes().contains(this.getTypeName()) : false; } @@ -185,40 +194,47 @@ public class AtlasClassificationType extends AtlasStructType { } } - private Set<String> getAllSuperTypes(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { - Set<String> superTypes = new HashSet<>(); - List<String> subTypes = new ArrayList<>(); - - collectAllSuperTypes(subTypes, superTypes, typeRegistry); + private void getTypeHierarchyInfo(AtlasTypeRegistry typeRegistry, + Set<String> allSuperTypeNames, + Map<String, AtlasAttributeDef> allAttributeDefs) throws AtlasBaseException { + List<String> visitedTypes = new ArrayList<>(); - return superTypes; + collectTypeHierarchyInfo(typeRegistry, allSuperTypeNames, allAttributeDefs, visitedTypes); } /* * This method should not assume that resolveReferences() has been called on all superTypes. * this.classificationDef is the only safe member to reference here */ - private void collectAllSuperTypes(List<String> subTypes, Set<String> superTypes, AtlasTypeRegistry typeRegistry) - throws AtlasBaseException { - if (subTypes.contains(classificationDef.getName())) { - throw new AtlasBaseException(AtlasErrorCode.CIRCULAR_REFERENCE, - classificationDef.getName(), subTypes.toString()); + private void collectTypeHierarchyInfo(AtlasTypeRegistry typeRegistry, + Set<String> allSuperTypeNames, + Map<String, AtlasAttributeDef> allAttributeDefs, + List<String> visitedTypes) throws AtlasBaseException { + if (visitedTypes.contains(classificationDef.getName())) { + throw new AtlasBaseException(AtlasErrorCode.CIRCULAR_REFERENCE, classificationDef.getName(), + visitedTypes.toString()); } if (CollectionUtils.isNotEmpty(classificationDef.getSuperTypes())) { - superTypes.addAll(classificationDef.getSuperTypes()); - - subTypes.add(classificationDef.getName()); + visitedTypes.add(classificationDef.getName()); for (String superTypeName : classificationDef.getSuperTypes()) { AtlasType type = typeRegistry.getType(superTypeName); if (type instanceof AtlasClassificationType) { AtlasClassificationType superType = (AtlasClassificationType) type; - superType.collectAllSuperTypes(subTypes, superTypes, typeRegistry); + superType.collectTypeHierarchyInfo(typeRegistry, allSuperTypeNames, allAttributeDefs, visitedTypes); } } - subTypes.remove(classificationDef.getName()); + visitedTypes.remove(classificationDef.getName()); + + allSuperTypeNames.addAll(classificationDef.getSuperTypes()); + } + + if (CollectionUtils.isNotEmpty(classificationDef.getAttributeDefs())) { + for (AtlasAttributeDef attributeDef : classificationDef.getAttributeDefs()) { + allAttributeDefs.put(attributeDef.getName(), attributeDef); + } } } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6a24cad1/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java index 9c5ec62..7170ee4 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java @@ -22,18 +22,19 @@ import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.typedef.AtlasEntityDef; +import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; - /** * class that implements behaviour of an entity-type. */ @@ -42,8 +43,9 @@ public class AtlasEntityType extends AtlasStructType { private final AtlasEntityDef entityDef; - private List<AtlasEntityType> superTypes = Collections.emptyList(); - private Set<String> allSuperTypes = Collections.emptySet(); + private List<AtlasEntityType> superTypes = Collections.emptyList(); + private Set<String> allSuperTypes = Collections.emptySet(); + private Map<String, AtlasAttributeDef> allAttributeDefs = Collections.emptyMap(); public AtlasEntityType(AtlasEntityDef entityDef) { super(entityDef, TypeCategory.ENTITY); @@ -63,8 +65,11 @@ public class AtlasEntityType extends AtlasStructType { public void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { super.resolveReferences(typeRegistry); - List<AtlasEntityType> s = new ArrayList<AtlasEntityType>(); - Set<String> allS = getAllSuperTypes(typeRegistry); + List<AtlasEntityType> s = new ArrayList<>(); + Set<String> allS = new HashSet<>(); + Map<String, AtlasAttributeDef> allA = new HashMap<>(); + + getTypeHierarchyInfo(typeRegistry, allS, allA); for (String superTypeName : entityDef.getSuperTypes()) { AtlasType superType = typeRegistry.getType(superTypeName); @@ -77,8 +82,9 @@ public class AtlasEntityType extends AtlasStructType { } } - this.superTypes = Collections.unmodifiableList(s); - this.allSuperTypes = Collections.unmodifiableSet(allS); + this.superTypes = Collections.unmodifiableList(s); + this.allSuperTypes = Collections.unmodifiableSet(allS); + this.allAttributeDefs = Collections.unmodifiableMap(allA); } public Set<String> getSuperTypes() { @@ -89,6 +95,8 @@ public class AtlasEntityType extends AtlasStructType { return allSuperTypes; } + public Map<String, AtlasAttributeDef> getAllAttributeDefs() { return allAttributeDefs; } + public boolean isSuperTypeOf(AtlasEntityType entityType) { return entityType != null ? entityType.getAllSuperTypes().contains(this.getTypeName()) : false; } @@ -186,39 +194,47 @@ public class AtlasEntityType extends AtlasStructType { } } - private Set<String> getAllSuperTypes(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { - Set<String> superTypes = new HashSet<>(); - List<String> subTypes = new ArrayList<>(); + private void getTypeHierarchyInfo(AtlasTypeRegistry typeRegistry, + Set<String> allSuperTypeNames, + Map<String, AtlasAttributeDef> allAttributeDefs) throws AtlasBaseException { + List<String> visitedTypes = new ArrayList<>(); - collectAllSuperTypes(subTypes, superTypes, typeRegistry); - - return superTypes; + collectTypeHierarchyInfo(typeRegistry, allSuperTypeNames, allAttributeDefs, visitedTypes); } /* * This method should not assume that resolveReferences() has been called on all superTypes. * this.entityDef is the only safe member to reference here */ - private void collectAllSuperTypes(List<String> subTypes, Set<String> superTypes, AtlasTypeRegistry typeRegistry) - throws AtlasBaseException { - if (subTypes.contains(entityDef.getName())) { - throw new AtlasBaseException(AtlasErrorCode.CIRCULAR_REFERENCE, entityDef.getName(), subTypes.toString()); + private void collectTypeHierarchyInfo(AtlasTypeRegistry typeRegistry, + Set<String> allSuperTypeNames, + Map<String, AtlasAttributeDef> allAttributeDefs, + List<String> visitedTypes) throws AtlasBaseException { + if (visitedTypes.contains(entityDef.getName())) { + throw new AtlasBaseException(AtlasErrorCode.CIRCULAR_REFERENCE, entityDef.getName(), + visitedTypes.toString()); } if (CollectionUtils.isNotEmpty(entityDef.getSuperTypes())) { - superTypes.addAll(entityDef.getSuperTypes()); - - subTypes.add(entityDef.getName()); + visitedTypes.add(entityDef.getName()); for (String superTypeName : entityDef.getSuperTypes()) { AtlasType type = typeRegistry.getType(superTypeName); if (type instanceof AtlasEntityType) { AtlasEntityType superType = (AtlasEntityType) type; - superType.collectAllSuperTypes(subTypes, superTypes, typeRegistry); + superType.collectTypeHierarchyInfo(typeRegistry, allSuperTypeNames, allAttributeDefs, visitedTypes); } } - subTypes.remove(entityDef.getName()); + visitedTypes.remove(entityDef.getName()); + + allSuperTypeNames.addAll(entityDef.getSuperTypes()); + } + + if (CollectionUtils.isNotEmpty(entityDef.getAttributeDefs())) { + for (AtlasAttributeDef attributeDef : entityDef.getAttributeDefs()) { + allAttributeDefs.put(attributeDef.getName(), attributeDef); + } } } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6a24cad1/intg/src/test/java/org/apache/atlas/type/TestAtlasTypeRegistry.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/type/TestAtlasTypeRegistry.java b/intg/src/test/java/org/apache/atlas/type/TestAtlasTypeRegistry.java index 7ad9a18..9429c07 100644 --- a/intg/src/test/java/org/apache/atlas/type/TestAtlasTypeRegistry.java +++ b/intg/src/test/java/org/apache/atlas/type/TestAtlasTypeRegistry.java @@ -19,14 +19,20 @@ package org.apache.atlas.type; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import org.apache.atlas.model.typedef.AtlasClassificationDef; import org.apache.atlas.model.typedef.AtlasEntityDef; +import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.type.AtlasTypeRegistry.AtlasTransientTypeRegistry; import org.testng.annotations.Test; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertNull; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import static org.testng.Assert.*; public class TestAtlasTypeRegistry { @@ -57,6 +63,14 @@ public class TestAtlasTypeRegistry { classifiL2_3.addSuperType(classifiL1_2.getName()); classifiL2_4.addSuperType(classifiL1_2.getName()); + classifiL0.addAttribute(new AtlasAttributeDef("L0_a1", AtlasBaseTypeDef.ATLAS_TYPE_INT)); + classifiL1_1.addAttribute(new AtlasAttributeDef("L1-1_a1", AtlasBaseTypeDef.ATLAS_TYPE_INT)); + classifiL1_2.addAttribute(new AtlasAttributeDef("L1-2_a1", AtlasBaseTypeDef.ATLAS_TYPE_INT)); + classifiL2_1.addAttribute(new AtlasAttributeDef("L2-1_a1", AtlasBaseTypeDef.ATLAS_TYPE_INT)); + classifiL2_2.addAttribute(new AtlasAttributeDef("L2-2_a1", AtlasBaseTypeDef.ATLAS_TYPE_INT)); + classifiL2_3.addAttribute(new AtlasAttributeDef("L2-3_a1", AtlasBaseTypeDef.ATLAS_TYPE_INT)); + classifiL2_4.addAttribute(new AtlasAttributeDef("L2-4_a1", AtlasBaseTypeDef.ATLAS_TYPE_INT)); + AtlasTypesDef typesDef = new AtlasTypesDef(); typesDef.getClassificationDefs().add(classifiL0); @@ -77,6 +91,24 @@ public class TestAtlasTypeRegistry { failureMsg = excp.getMessage(); } assertNull(failureMsg); + + typeRegistry.commitTransientTypeRegistry(ttr); + + validateSuperTypes(typeRegistry, "L0", new HashSet<String>()); + validateSuperTypes(typeRegistry, "L1-1", new HashSet<>(Arrays.asList("L0"))); + validateSuperTypes(typeRegistry, "L1-2", new HashSet<>(Arrays.asList("L0"))); + validateSuperTypes(typeRegistry, "L2-1", new HashSet<>(Arrays.asList("L1-1", "L0"))); + validateSuperTypes(typeRegistry, "L2-2", new HashSet<>(Arrays.asList("L1-1", "L0"))); + validateSuperTypes(typeRegistry, "L2-3", new HashSet<>(Arrays.asList("L1-1", "L0", "L1-2"))); + validateSuperTypes(typeRegistry, "L2-4", new HashSet<>(Arrays.asList("L1-2", "L0"))); + + validateAttributeNames(typeRegistry, "L0", new HashSet<>(Arrays.asList("L0_a1"))); + validateAttributeNames(typeRegistry, "L1-1", new HashSet<>(Arrays.asList("L0_a1", "L1-1_a1"))); + validateAttributeNames(typeRegistry, "L1-2", new HashSet<>(Arrays.asList("L0_a1", "L1-2_a1"))); + validateAttributeNames(typeRegistry, "L2-1", new HashSet<>(Arrays.asList("L0_a1", "L1-1_a1", "L2-1_a1"))); + validateAttributeNames(typeRegistry, "L2-2", new HashSet<>(Arrays.asList("L0_a1", "L1-1_a1", "L2-2_a1"))); + validateAttributeNames(typeRegistry, "L2-3", new HashSet<>(Arrays.asList("L0_a1", "L1-1_a1", "L1-2_a1", "L2-3_a1"))); + validateAttributeNames(typeRegistry, "L2-4", new HashSet<>(Arrays.asList("L0_a1", "L1-2_a1", "L2-4_a1"))); } @Test @@ -176,6 +208,14 @@ public class TestAtlasTypeRegistry { entL2_3.addSuperType(entL1_2.getName()); entL2_4.addSuperType(entL1_2.getName()); + entL0.addAttribute(new AtlasAttributeDef("L0_a1", AtlasBaseTypeDef.ATLAS_TYPE_INT)); + entL1_1.addAttribute(new AtlasAttributeDef("L1-1_a1", AtlasBaseTypeDef.ATLAS_TYPE_INT)); + entL1_2.addAttribute(new AtlasAttributeDef("L1-2_a1", AtlasBaseTypeDef.ATLAS_TYPE_INT)); + entL2_1.addAttribute(new AtlasAttributeDef("L2-1_a1", AtlasBaseTypeDef.ATLAS_TYPE_INT)); + entL2_2.addAttribute(new AtlasAttributeDef("L2-2_a1", AtlasBaseTypeDef.ATLAS_TYPE_INT)); + entL2_3.addAttribute(new AtlasAttributeDef("L2-3_a1", AtlasBaseTypeDef.ATLAS_TYPE_INT)); + entL2_4.addAttribute(new AtlasAttributeDef("L2-4_a1", AtlasBaseTypeDef.ATLAS_TYPE_INT)); + AtlasTypesDef typesDef = new AtlasTypesDef(); typesDef.getEntityDefs().add(entL0); @@ -196,6 +236,24 @@ public class TestAtlasTypeRegistry { failureMsg = excp.getMessage(); } assertNull(failureMsg); + + typeRegistry.commitTransientTypeRegistry(ttr); + + validateSuperTypes(typeRegistry, "L0", new HashSet<String>()); + validateSuperTypes(typeRegistry, "L1-1", new HashSet<>(Arrays.asList("L0"))); + validateSuperTypes(typeRegistry, "L1-2", new HashSet<>(Arrays.asList("L0"))); + validateSuperTypes(typeRegistry, "L2-1", new HashSet<>(Arrays.asList("L1-1", "L0"))); + validateSuperTypes(typeRegistry, "L2-2", new HashSet<>(Arrays.asList("L1-1", "L0"))); + validateSuperTypes(typeRegistry, "L2-3", new HashSet<>(Arrays.asList("L1-1", "L0", "L1-2"))); + validateSuperTypes(typeRegistry, "L2-4", new HashSet<>(Arrays.asList("L1-2", "L0"))); + + validateAttributeNames(typeRegistry, "L0", new HashSet<>(Arrays.asList("L0_a1"))); + validateAttributeNames(typeRegistry, "L1-1", new HashSet<>(Arrays.asList("L0_a1", "L1-1_a1"))); + validateAttributeNames(typeRegistry, "L1-2", new HashSet<>(Arrays.asList("L0_a1", "L1-2_a1"))); + validateAttributeNames(typeRegistry, "L2-1", new HashSet<>(Arrays.asList("L0_a1", "L1-1_a1", "L2-1_a1"))); + validateAttributeNames(typeRegistry, "L2-2", new HashSet<>(Arrays.asList("L0_a1", "L1-1_a1", "L2-2_a1"))); + validateAttributeNames(typeRegistry, "L2-3", new HashSet<>(Arrays.asList("L0_a1", "L1-1_a1", "L1-2_a1", "L2-3_a1"))); + validateAttributeNames(typeRegistry, "L2-4", new HashSet<>(Arrays.asList("L0_a1", "L1-2_a1", "L2-4_a1"))); } @Test @@ -267,4 +325,47 @@ public class TestAtlasTypeRegistry { } assertNotNull(failureMsg, "expected invalid supertype failure"); } + + private void validateSuperTypes(AtlasTypeRegistry typeRegistry, String typeName, Set<String> expectedSuperTypes) { + AtlasType type = null; + + try { + type = typeRegistry.getType(typeName); + } catch (AtlasBaseException excp) { + } + + Set<String> superTypes = null; + + if (type != null) { + if (type instanceof AtlasEntityType) { + superTypes = ((AtlasEntityType) type).getAllSuperTypes(); + } else if (type instanceof AtlasClassificationType) { + superTypes = ((AtlasClassificationType) type).getAllSuperTypes(); + } + } + + assertEquals(superTypes, expectedSuperTypes); + } + + private void validateAttributeNames(AtlasTypeRegistry typeRegistry, String typeName, Set<String> attributeNames) { + AtlasType type = null; + + try { + type = typeRegistry.getType(typeName); + } catch (AtlasBaseException excp) { + } + + Map<String, AtlasAttributeDef> attributeDefs = null; + + if (type != null) { + if (type instanceof AtlasEntityType) { + attributeDefs = ((AtlasEntityType) type).getAllAttributeDefs(); + } else if (type instanceof AtlasClassificationType) { + attributeDefs = ((AtlasClassificationType) type).getAllAttributeDefs(); + } + } + + assertNotNull(attributeDefs); + assertEquals(attributeDefs.keySet(), attributeNames); + } }
