ATLAS-1230: updated AtlasTypeRegistry to support batch, atomic type updates
Signed-off-by: Suma Shivaprasad <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/eb6e656b Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/eb6e656b Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/eb6e656b Branch: refs/heads/master Commit: eb6e656be868fba22aaa330d98137be78e30bdcb Parents: 4d9cf45 Author: Madhan Neethiraj <[email protected]> Authored: Tue Oct 18 15:27:46 2016 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Tue Oct 18 18:33:36 2016 -0700 ---------------------------------------------------------------------- .../org/apache/atlas/type/AtlasArrayType.java | 10 - .../atlas/type/AtlasClassificationType.java | 11 +- .../org/apache/atlas/type/AtlasEntityType.java | 11 +- .../org/apache/atlas/type/AtlasMapType.java | 19 - .../org/apache/atlas/type/AtlasStructType.java | 13 +- .../apache/atlas/type/AtlasTypeRegistry.java | 847 +++++++++++-------- .../org/apache/atlas/model/ModelTestUtil.java | 55 +- .../model/instance/TestAtlasClassification.java | 7 +- .../atlas/model/instance/TestAtlasEntity.java | 7 +- .../atlas/model/typedef/TestAtlasEntityDef.java | 7 +- .../apache/atlas/type/TestAtlasEntityType.java | 33 +- release-log.txt | 1 + .../store/graph/AtlasTypeDefGraphStore.java | 292 ++++--- .../store/graph/v1/AtlasAbstractDefStoreV1.java | 33 + .../graph/v1/AtlasClassificationDefStoreV1.java | 95 +-- .../store/graph/v1/AtlasEntityDefStoreV1.java | 94 +- .../store/graph/v1/AtlasEnumDefStoreV1.java | 87 +- .../store/graph/v1/AtlasStructDefStoreV1.java | 214 +++-- .../graph/v1/AtlasTypeDefGraphStoreV1.java | 36 +- .../org/apache/atlas/web/rest/TypesREST.java | 2 +- 20 files changed, 1038 insertions(+), 836 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/eb6e656b/intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java b/intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java index 21a4037..e8092a5 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java @@ -103,16 +103,6 @@ public class AtlasArrayType extends AtlasType { @Override public void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { elementType = typeRegistry.getType(elementTypeName); - - if (elementType == null) { - String msg = elementTypeName + ": unknown element-type for array"; - - LOG.error(msg); - - throw new AtlasBaseException(msg); - } - - elementType.resolveReferences(typeRegistry); } @Override http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/eb6e656b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java index d962bbe..6299265 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java @@ -70,7 +70,7 @@ public class AtlasClassificationType extends AtlasStructType { for (String superTypeName : classificationDef.getSuperTypes()) { AtlasType superType = typeRegistry.getType(superTypeName); - if (superType != null && superType instanceof AtlasClassificationType) { + if (superType instanceof AtlasClassificationType) { AtlasClassificationType superClassificationType = (AtlasClassificationType)superType; superClassificationType.resolveReferences(typeRegistry); @@ -82,13 +82,8 @@ public class AtlasClassificationType extends AtlasStructType { allS.addAll(superClassificationType.getAllSuperTypes()); } } else { - String msg = superTypeName + ((superType == null) ? ": unknown" : ": incompatible"); - - msg += (" supertype in classification " + classificationDef.getName()); - - LOG.error(msg); - - throw new AtlasBaseException(msg); + throw new AtlasBaseException(superTypeName + ": incompatible supertype in classification " + + classificationDef.getName()); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/eb6e656b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java index 7017d65..9ae61a7 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java @@ -68,7 +68,7 @@ public class AtlasEntityType extends AtlasStructType { for (String superTypeName : entityDef.getSuperTypes()) { AtlasType superType = typeRegistry.getType(superTypeName); - if (superType != null && superType instanceof AtlasEntityType) { + if (superType instanceof AtlasEntityType) { AtlasEntityType superEntityType = (AtlasEntityType)superType; superEntityType.resolveReferences(typeRegistry); @@ -80,13 +80,8 @@ public class AtlasEntityType extends AtlasStructType { allS.addAll(superEntityType.getAllSuperTypes()); } } else { - String msg = superTypeName + ((superType == null) ? ": unknown" : ": incompatible"); - - msg += (" supertype in entity " + entityDef.getName()); - - LOG.error(msg); - - throw new AtlasBaseException(msg); + throw new AtlasBaseException(superTypeName + ": incompatible supertype in entity " + + entityDef.getName()); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/eb6e656b/intg/src/main/java/org/apache/atlas/type/AtlasMapType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasMapType.java b/intg/src/main/java/org/apache/atlas/type/AtlasMapType.java index f2ab08b..0ebfa26 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasMapType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasMapType.java @@ -88,25 +88,6 @@ public class AtlasMapType extends AtlasType { public void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { this.keyType = typeRegistry.getType(keyTypeName); this.valueType = typeRegistry.getType(valueTypeName); - - if (keyType == null) { - String msg = keyTypeName + ": unknown key-type for map"; - - LOG.error(msg); - - throw new AtlasBaseException(msg); - } - - if (valueType == null) { - String msg = valueTypeName + ": unknown value-type for map"; - - LOG.error(msg); - - throw new AtlasBaseException(msg); - } - - keyType.resolveReferences(typeRegistry); - valueType.resolveReferences(typeRegistry); } @Override http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/eb6e656b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java index 5c32db2..4e741e5 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java @@ -103,11 +103,6 @@ public class AtlasStructType extends AtlasType { for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) { AtlasType attrType = typeRegistry.getType(attributeDef.getTypeName()); - if (attrType == null) { - throw new AtlasBaseException(attributeDef.getTypeName() + ": unknown type for attribute " - + structDef.getName() + "." + attributeDef.getName()); - } - resolveConstraints(attributeDef, attrType); Cardinality cardinality = attributeDef.getCardinality(); @@ -407,8 +402,8 @@ public class AtlasStructType extends AtlasType { if (StringUtils.isBlank(refAttribName)) { throw new AtlasBaseException(getTypeName() + "." + attribDef.getName() + ": " - + CONSTRAINT_TYPE_MAPPED_FROM_REF + " invalid constraint. missing parameter " - + CONSTRAINT_PARAM_REF_ATTRIBUTE); + + " invalid constraint. missing parameter " + CONSTRAINT_PARAM_REF_ATTRIBUTE + + " in " + CONSTRAINT_TYPE_MAPPED_FROM_REF + ". params=" + constraintDef.getParams()); } AtlasStructType structType = (AtlasStructType)attribType; @@ -421,8 +416,8 @@ public class AtlasStructType extends AtlasType { } if (!StringUtils.equals(getTypeName(), refAttrib.getTypeName())) { - throw new AtlasBaseException(getTypeName() + "." + attribDef.getName() + ": invalid constraint. Datatype of" - + CONSTRAINT_PARAM_REF_ATTRIBUTE + " " + structType.getTypeName() + "." + refAttribName + throw new AtlasBaseException(getTypeName() + "." + attribDef.getName() + ": invalid constraint. Datatype" + + " of " + CONSTRAINT_PARAM_REF_ATTRIBUTE + " " + structType.getTypeName() + "." + refAttribName + " should be " + getTypeName() + ", but found " + refAttrib.getTypeName()); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/eb6e656b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java index c3f17f4..f31ee50 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java @@ -17,12 +17,15 @@ */ package org.apache.atlas.type; +import com.sun.jersey.spi.resource.Singleton; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import org.apache.atlas.model.typedef.AtlasClassificationDef; -import org.apache.atlas.model.typedef.AtlasEnumDef; import org.apache.atlas.model.typedef.AtlasEntityDef; +import org.apache.atlas.model.typedef.AtlasEnumDef; import org.apache.atlas.model.typedef.AtlasStructDef; +import org.apache.atlas.model.typedef.AtlasTypesDef; + import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_ARRAY_PREFIX; import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_ARRAY_SUFFIX; import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_MAP_PREFIX; @@ -37,532 +40,652 @@ import org.slf4j.LoggerFactory; import java.util.Collection; import java.util.Collections; import java.util.Map; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; /** * registry for all types defined in Atlas. */ +@Singleton public class AtlasTypeRegistry { private static final Logger LOG = LoggerFactory.getLogger(AtlasStructType.class); - private final Map<String, AtlasType> allTypes; - private final TypeDefCache<AtlasEnumDef> enumDefs; - private final TypeDefCache<AtlasStructDef> structDefs; - private final TypeDefCache<AtlasClassificationDef> classificationDefs; - private final TypeDefCache<AtlasEntityDef> entityDefs; + protected RegistryData registryData; public AtlasTypeRegistry() { - allTypes = new ConcurrentHashMap<>(); - enumDefs = new TypeDefCache<>(this); - structDefs = new TypeDefCache<>(this); - classificationDefs = new TypeDefCache<>(this); - entityDefs = new TypeDefCache<>(this); - - registerType(new AtlasBuiltInTypes.AtlasBooleanType()); - registerType(new AtlasBuiltInTypes.AtlasByteType()); - registerType(new AtlasBuiltInTypes.AtlasShortType()); - registerType(new AtlasBuiltInTypes.AtlasIntType()); - registerType(new AtlasBuiltInTypes.AtlasLongType()); - registerType(new AtlasBuiltInTypes.AtlasFloatType()); - registerType(new AtlasBuiltInTypes.AtlasDoubleType()); - registerType(new AtlasBuiltInTypes.AtlasBigIntegerType()); - registerType(new AtlasBuiltInTypes.AtlasBigDecimalType()); - registerType(new AtlasBuiltInTypes.AtlasDateType()); - registerType(new AtlasBuiltInTypes.AtlasStringType()); - registerType(new AtlasBuiltInTypes.AtlasObjectIdType()); - } - - public Collection<String> getAllTypeNames() { return Collections.unmodifiableSet(allTypes.keySet()); } - - public AtlasType getType(String typeName) { + registryData = new RegistryData(); + } + + protected AtlasTypeRegistry(AtlasTypeRegistry other) { + registryData = new RegistryData(other.registryData); + } + + public Collection<String> getAllTypeNames() { return registryData.allTypes.getAllTypeNames(); } + + public AtlasType getType(String typeName) throws AtlasBaseException { if (LOG.isDebugEnabled()) { LOG.debug("==> AtlasTypeRegistry.getType({})", typeName); } - AtlasType ret = allTypes.get(typeName); + AtlasType ret = registryData.allTypes.getTypeByName(typeName); if (ret == null) { - try { - if (typeName.startsWith(ATLAS_TYPE_ARRAY_PREFIX) && typeName.endsWith(ATLAS_TYPE_ARRAY_SUFFIX)) { - int startIdx = ATLAS_TYPE_ARRAY_PREFIX.length(); - int endIdx = typeName.length() - ATLAS_TYPE_ARRAY_SUFFIX.length(); - String elementTypeName = typeName.substring(startIdx, endIdx); - - ret = new AtlasArrayType(elementTypeName, this); - } else if (typeName.startsWith(ATLAS_TYPE_MAP_PREFIX) && typeName.endsWith(ATLAS_TYPE_MAP_SUFFIX)) { - int startIdx = ATLAS_TYPE_MAP_PREFIX.length(); - int endIdx = typeName.length() - ATLAS_TYPE_MAP_SUFFIX.length(); - String[] keyValueTypes = typeName.substring(startIdx, endIdx).split(ATLAS_TYPE_MAP_KEY_VAL_SEP, 2); - String keyTypeName = keyValueTypes.length > 0 ? keyValueTypes[0] : null; - String valueTypeName = keyValueTypes.length > 1 ? keyValueTypes[1] : null; - - ret = new AtlasMapType(keyTypeName, valueTypeName, this); - } - } catch(AtlasBaseException excp) { - LOG.warn("failed to instantiate type for " + typeName, excp); + if (typeName.startsWith(ATLAS_TYPE_ARRAY_PREFIX) && typeName.endsWith(ATLAS_TYPE_ARRAY_SUFFIX)) { + int startIdx = ATLAS_TYPE_ARRAY_PREFIX.length(); + int endIdx = typeName.length() - ATLAS_TYPE_ARRAY_SUFFIX.length(); + String elementTypeName = typeName.substring(startIdx, endIdx); + + ret = new AtlasArrayType(elementTypeName, this); + } else if (typeName.startsWith(ATLAS_TYPE_MAP_PREFIX) && typeName.endsWith(ATLAS_TYPE_MAP_SUFFIX)) { + int startIdx = ATLAS_TYPE_MAP_PREFIX.length(); + int endIdx = typeName.length() - ATLAS_TYPE_MAP_SUFFIX.length(); + String[] keyValueTypes = typeName.substring(startIdx, endIdx).split(ATLAS_TYPE_MAP_KEY_VAL_SEP, 2); + String keyTypeName = keyValueTypes.length > 0 ? keyValueTypes[0] : null; + String valueTypeName = keyValueTypes.length > 1 ? keyValueTypes[1] : null; + + ret = new AtlasMapType(keyTypeName, valueTypeName, this); + } else { + throw new AtlasBaseException(typeName + ": unknown typename"); } } if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.getType({})", typeName); + LOG.debug("<== AtlasTypeRegistry.getType({}): {}", typeName, ret); } return ret; } - - public void addType(AtlasBaseTypeDef typeDef) throws AtlasBaseException { + public AtlasType getTypeByGuid(String guid) { if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.addType({})", typeDef); + LOG.debug("==> AtlasTypeRegistry.getTypeByGuid({})", guid); } - if (typeDef == null) { - // ignore - } else if (typeDef.getClass().equals(AtlasEnumDef.class)) { - AtlasEnumDef enumDef = (AtlasEnumDef)typeDef; + AtlasType ret = registryData.allTypes.getTypeByGuid(guid); - enumDefs.addType(enumDef, new AtlasEnumType(enumDef)); - } else if (typeDef.getClass().equals(AtlasStructDef.class)) { - AtlasStructDef structDef = (AtlasStructDef)typeDef; + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasTypeRegistry.getTypeByGuid({}): {}", guid, ret); + } - structDefs.addType(structDef, new AtlasStructType(structDef, this)); - } else if (typeDef.getClass().equals(AtlasClassificationDef.class)) { - AtlasClassificationDef classificationDef = (AtlasClassificationDef)typeDef; + return ret; + } - classificationDefs.addType(classificationDef, new AtlasClassificationType(classificationDef, this)); - } else if (typeDef.getClass().equals(AtlasEntityDef.class)) { - AtlasEntityDef entityDef = (AtlasEntityDef)typeDef; - entityDefs.addType(entityDef, new AtlasEntityType(entityDef, this)); - } + public Collection<AtlasEnumDef> getAllEnumDefs() { return registryData.enumDefs.getAll(); } - if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.addType({})", typeDef); - } + public AtlasEnumDef getEnumDefByGuid(String guid) { + return registryData.enumDefs.getTypeDefByGuid(guid); } - public void addTypeWithNoRefResolve(AtlasBaseTypeDef typeDef) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.addTypeWithNoRefResolve({})", typeDef); - } + public AtlasEnumDef getEnumDefByName(String name) { + return registryData.enumDefs.getTypeDefByName(name); + } - if (typeDef == null) { - // ignore - } else if (typeDef.getClass().equals(AtlasEnumDef.class)) { - AtlasEnumDef enumDef = (AtlasEnumDef)typeDef; - enumDefs.addType(enumDef, new AtlasEnumType(enumDef)); - } else if (typeDef.getClass().equals(AtlasStructDef.class)) { - AtlasStructDef structDef = (AtlasStructDef)typeDef; + public Collection<AtlasStructDef> getAllStructDefs() { return registryData.structDefs.getAll(); } - structDefs.addType(structDef, new AtlasStructType(structDef)); - } else if (typeDef.getClass().equals(AtlasClassificationDef.class)) { - AtlasClassificationDef classificationDef = (AtlasClassificationDef)typeDef; + public AtlasStructDef getStructDefByGuid(String guid) { + return registryData.structDefs.getTypeDefByGuid(guid); + } - classificationDefs.addType(classificationDef, new AtlasClassificationType(classificationDef)); - } else if (typeDef.getClass().equals(AtlasEntityDef.class)) { - AtlasEntityDef entityDef = (AtlasEntityDef)typeDef; + public AtlasStructDef getStructDefByName(String name) { return registryData.structDefs.getTypeDefByName(name); } - entityDefs.addType(entityDef, new AtlasEntityType(entityDef)); - } - if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.addTypeWithNoRefResolve({})", typeDef); - } + public Collection<AtlasClassificationDef> getAllClassificationDefs() { + return registryData.classificationDefs.getAll(); } - public void updateType(AtlasBaseTypeDef typeDef) throws AtlasBaseException { - if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.updateType({})", typeDef); - } + public AtlasClassificationDef getClassificationDefByGuid(String guid) { + return registryData.classificationDefs.getTypeDefByGuid(guid); + } - if (typeDef == null) { - // ignore - } else if (StringUtils.isNotBlank(typeDef.getGuid())) { - updateTypeByGuid(typeDef.getGuid(), typeDef); - } else if (StringUtils.isNotBlank(typeDef.getName())) { - updateTypeByName(typeDef.getName(), typeDef); - } + public AtlasClassificationDef getClassificationDefByName(String name) { + return registryData.classificationDefs.getTypeDefByName(name); + } - if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.updateType({})", typeDef); - } + + public Collection<AtlasEntityDef> getAllEntityDefs() { return registryData.entityDefs.getAll(); } + + public AtlasEntityDef getEntityDefByGuid(String guid) { + return registryData.entityDefs.getTypeDefByGuid(guid); } - public void updateTypeWithNoRefResolve(AtlasBaseTypeDef typeDef) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.updateType({})", typeDef); - } + public AtlasEntityDef getEntityDefByName(String name) { + return registryData.entityDefs.getTypeDefByName(name); + } - if (typeDef == null) { - // ignore - } else if (StringUtils.isNotBlank(typeDef.getGuid())) { - updateTypeByGuidWithNoRefResolve(typeDef.getGuid(), typeDef); - } else if (StringUtils.isNotBlank(typeDef.getName())) { - updateTypeByNameWithNoRefResolve(typeDef.getName(), typeDef); - } + public AtlasTransientTypeRegistry createTransientTypeRegistry() { + return new AtlasTransientTypeRegistry(this); + } - if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.updateType({})", typeDef); + public void commitTransientTypeRegistry(AtlasTransientTypeRegistry transientTypeRegistry) { + this.registryData = transientTypeRegistry.registryData; + } + + static class RegistryData { + final TypeCache allTypes; + final TypeDefCache<AtlasEnumDef> enumDefs; + final TypeDefCache<AtlasStructDef> structDefs; + final TypeDefCache<AtlasClassificationDef> classificationDefs; + final TypeDefCache<AtlasEntityDef> entityDefs; + + RegistryData() { + allTypes = new TypeCache(); + enumDefs = new TypeDefCache<>(allTypes); + structDefs = new TypeDefCache<>(allTypes); + classificationDefs = new TypeDefCache<>(allTypes); + entityDefs = new TypeDefCache<>(allTypes); + + allTypes.addType(new AtlasBuiltInTypes.AtlasBooleanType()); + allTypes.addType(new AtlasBuiltInTypes.AtlasByteType()); + allTypes.addType(new AtlasBuiltInTypes.AtlasShortType()); + allTypes.addType(new AtlasBuiltInTypes.AtlasIntType()); + allTypes.addType(new AtlasBuiltInTypes.AtlasLongType()); + allTypes.addType(new AtlasBuiltInTypes.AtlasFloatType()); + allTypes.addType(new AtlasBuiltInTypes.AtlasDoubleType()); + allTypes.addType(new AtlasBuiltInTypes.AtlasBigIntegerType()); + allTypes.addType(new AtlasBuiltInTypes.AtlasBigDecimalType()); + allTypes.addType(new AtlasBuiltInTypes.AtlasDateType()); + allTypes.addType(new AtlasBuiltInTypes.AtlasStringType()); + allTypes.addType(new AtlasBuiltInTypes.AtlasObjectIdType()); + } + + RegistryData(RegistryData other) { + allTypes = new TypeCache(other.allTypes); + enumDefs = new TypeDefCache<>(other.enumDefs, allTypes); + structDefs = new TypeDefCache<>(other.structDefs, allTypes); + classificationDefs = new TypeDefCache<>(other.classificationDefs, allTypes); + entityDefs = new TypeDefCache<>(other.entityDefs, allTypes); } } - public void updateTypeByGuid(String guid, AtlasBaseTypeDef typeDef) throws AtlasBaseException { - if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.updateTypeByGuid({})", guid); + public static class AtlasTransientTypeRegistry extends AtlasTypeRegistry { + + private AtlasTransientTypeRegistry(AtlasTypeRegistry parent) { + super(parent); } - if (guid == null || typeDef == null) { - // ignore - } else if (typeDef.getClass().equals(AtlasEnumDef.class)) { - AtlasEnumDef enumDef = (AtlasEnumDef)typeDef; + private void resolveReferences() throws AtlasBaseException { + for (AtlasType type : registryData.allTypes.getAllTypes()) { + type.resolveReferences(this); + } + } - enumDefs.removeTypeDefByGuid(guid); - enumDefs.addType(enumDef, new AtlasEnumType(enumDef)); - } else if (typeDef.getClass().equals(AtlasStructDef.class)) { - AtlasStructDef structDef = (AtlasStructDef)typeDef; + public void addType(AtlasBaseTypeDef typeDef) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasTypeRegistry.addType({})", typeDef); + } - structDefs.removeTypeDefByGuid(guid); - structDefs.addType(structDef, new AtlasStructType(structDef, this)); - } else if (typeDef.getClass().equals(AtlasClassificationDef.class)) { - AtlasClassificationDef classificationDef = (AtlasClassificationDef)typeDef; + if (typeDef != null) { + addTypeWithNoRefResolve(typeDef); - classificationDefs.removeTypeDefByGuid(guid); - classificationDefs.addType(classificationDef, new AtlasClassificationType(classificationDef, this)); - } else if (typeDef.getClass().equals(AtlasEntityDef.class)) { - AtlasEntityDef entityDef = (AtlasEntityDef)typeDef; + resolveReferences(); + } - entityDefs.removeTypeDefByGuid(guid); - entityDefs.addType(entityDef, new AtlasEntityType(entityDef, this)); + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasTypeRegistry.addType({})", typeDef); + } } - if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.updateTypeByGuid({})", guid); - } - } + public void addTypes(Collection<? extends AtlasBaseTypeDef> typeDefs) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasTypeRegistry.addTypes(length={})", (typeDefs == null ? 0 : typeDefs.size())); + } - public void updateTypeByName(String name, AtlasBaseTypeDef typeDef) throws AtlasBaseException { - if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.updateEnumDefByName({})", name); + if (CollectionUtils.isNotEmpty(typeDefs)) { + addTypesWithNoRefResolve(typeDefs); + + resolveReferences(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasTypeRegistry.addTypes(length={})", (typeDefs == null ? 0 : typeDefs.size())); + } } - if (name == null || typeDef == null) { - // ignore - } else if (typeDef.getClass().equals(AtlasEnumDef.class)) { - AtlasEnumDef enumDef = (AtlasEnumDef)typeDef; + public void addTypes(AtlasTypesDef typesDef) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasTypeRegistry.addTypes({})", typesDef); + } - enumDefs.removeTypeDefByName(name); - enumDefs.addType(enumDef, new AtlasEnumType(enumDef)); - } else if (typeDef.getClass().equals(AtlasStructDef.class)) { - AtlasStructDef structDef = (AtlasStructDef)typeDef; + if (typesDef != null) { + addTypesWithNoRefResolve(typesDef.getEnumDefs()); + addTypesWithNoRefResolve(typesDef.getStructDefs()); + addTypesWithNoRefResolve(typesDef.getClassificationDefs()); + addTypesWithNoRefResolve(typesDef.getEntityDefs()); - structDefs.removeTypeDefByName(name); - structDefs.addType(structDef, new AtlasStructType(structDef, this)); - } else if (typeDef.getClass().equals(AtlasClassificationDef.class)) { - AtlasClassificationDef classificationDef = (AtlasClassificationDef)typeDef; + resolveReferences(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasTypeRegistry.addTypes({})", typesDef); + } + } - classificationDefs.removeTypeDefByName(name); - classificationDefs.addType(classificationDef, new AtlasClassificationType(classificationDef, this)); - } else if (typeDef.getClass().equals(AtlasEntityDef.class)) { - AtlasEntityDef entityDef = (AtlasEntityDef)typeDef; + public void updateType(AtlasBaseTypeDef typeDef) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasTypeRegistry.updateType({})", typeDef); + } + + if (typeDef != null) { + updateTypeWithNoRefResolve(typeDef); - entityDefs.removeTypeDefByName(name); - entityDefs.addType(entityDef, new AtlasEntityType(entityDef, this)); + resolveReferences(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasTypeRegistry.updateType({})", typeDef); + } } - if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.updateEnumDefByName({})", name); + public void updateTypeByGuid(String guid, AtlasBaseTypeDef typeDef) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasTypeRegistry.updateTypeByGuid({})", guid); + } + + if (guid != null && typeDef != null) { + updateTypeByGuidWithNoRefResolve(guid, typeDef); + + resolveReferences(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasTypeRegistry.updateTypeByGuid({})", guid); + } } - } - public void updateTypeByGuidWithNoRefResolve(String guid, AtlasBaseTypeDef typeDef) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.updateTypeByGuidWithNoRefResolve({})", guid); + public void updateTypeByName(String name, AtlasBaseTypeDef typeDef) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasTypeRegistry.updateEnumDefByName({})", name); + } + + if (name != null && typeDef != null) { + updateTypeByNameWithNoRefResolve(name, typeDef); + + resolveReferences(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasTypeRegistry.updateEnumDefByName({})", name); + } } - if (guid == null || typeDef == null) { - // ignore - } else if (typeDef.getClass().equals(AtlasEnumDef.class)) { - AtlasEnumDef enumDef = (AtlasEnumDef)typeDef; + public void updateTypes(Collection<? extends AtlasBaseTypeDef> typeDefs) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasTypeRegistry.updateTypes(length={})", (typeDefs == null ? 0 : typeDefs.size())); + } - enumDefs.removeTypeDefByGuid(guid); - enumDefs.addType(enumDef, new AtlasEnumType(enumDef)); - } else if (typeDef.getClass().equals(AtlasStructDef.class)) { - AtlasStructDef structDef = (AtlasStructDef)typeDef; + if (CollectionUtils.isNotEmpty(typeDefs)) { + updateTypesWithNoRefResolve(typeDefs); - structDefs.removeTypeDefByGuid(guid); - structDefs.addType(structDef, new AtlasStructType(structDef)); - } else if (typeDef.getClass().equals(AtlasClassificationDef.class)) { - AtlasClassificationDef classificationDef = (AtlasClassificationDef)typeDef; + resolveReferences(); + } - classificationDefs.removeTypeDefByGuid(guid); - classificationDefs.addType(classificationDef, new AtlasClassificationType(classificationDef)); - } else if (typeDef.getClass().equals(AtlasEntityDef.class)) { - AtlasEntityDef entityDef = (AtlasEntityDef)typeDef; + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasTypeRegistry.updateTypes(length={})", (typeDefs == null ? 0 : typeDefs.size())); + } + } - entityDefs.removeTypeDefByGuid(guid); - entityDefs.addType(entityDef, new AtlasEntityType(entityDef)); + public void updateTypes(AtlasTypesDef typesDef) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasTypeRegistry.updateTypes({})", typesDef); + } + + if (typesDef != null) { + updateTypesWithNoRefResolve(typesDef.getEnumDefs()); + updateTypesWithNoRefResolve(typesDef.getStructDefs()); + updateTypesWithNoRefResolve(typesDef.getClassificationDefs()); + updateTypesWithNoRefResolve(typesDef.getEntityDefs()); + + resolveReferences(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasTypeRegistry.updateTypes({})", typesDef); + } } - if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.updateTypeByGuidWithNoRefResolve({})", guid); + public void removeTypeByGuid(String guid) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasTypeRegistry.removeTypeByGuid({})", guid); + } + + if (guid != null) { + registryData.enumDefs.removeTypeDefByGuid(guid); + registryData.structDefs.removeTypeDefByGuid(guid); + registryData.classificationDefs.removeTypeDefByGuid(guid); + registryData.entityDefs.removeTypeDefByGuid(guid); + + resolveReferences(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasTypeRegistry.removeTypeByGuid({})", guid); + } } - } - public void updateTypeByNameWithNoRefResolve(String name, AtlasBaseTypeDef typeDef) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.updateTypeByNameWithNoRefResolve({})", name); + public void removeTypeByName(String name) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasTypeRegistry.removeTypeByName({})", name); + } + + if (name != null) { + registryData.enumDefs.removeTypeDefByName(name); + registryData.structDefs.removeTypeDefByName(name); + registryData.classificationDefs.removeTypeDefByName(name); + registryData.entityDefs.removeTypeDefByName(name); + + resolveReferences(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasTypeRegistry.removeEnumDefByName({})", name); + } } - if (name == null || typeDef == null) { - // ignore - } else if (typeDef.getClass().equals(AtlasEnumDef.class)) { - AtlasEnumDef enumDef = (AtlasEnumDef)typeDef; - enumDefs.removeTypeDefByName(name); - enumDefs.addType(enumDef, new AtlasEnumType(enumDef)); - } else if (typeDef.getClass().equals(AtlasStructDef.class)) { - AtlasStructDef structDef = (AtlasStructDef)typeDef; + private void addTypeWithNoRefResolve(AtlasBaseTypeDef typeDef) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasTypeRegistry.addTypeWithNoRefResolve({})", typeDef); + } - structDefs.removeTypeDefByName(name); - structDefs.addType(structDef, new AtlasStructType(structDef)); - } else if (typeDef.getClass().equals(AtlasClassificationDef.class)) { - AtlasClassificationDef classificationDef = (AtlasClassificationDef)typeDef; + if (typeDef != null) { + if (StringUtils.isBlank(typeDef.getGuid())) { + typeDef.setGuid(UUID.randomUUID().toString()); + } - classificationDefs.removeTypeDefByName(name); - classificationDefs.addType(classificationDef, new AtlasClassificationType(classificationDef)); - } else if (typeDef.getClass().equals(AtlasEntityDef.class)) { - AtlasEntityDef entityDef = (AtlasEntityDef)typeDef; + if (typeDef.getClass().equals(AtlasEnumDef.class)) { + AtlasEnumDef enumDef = (AtlasEnumDef) typeDef; - entityDefs.removeTypeDefByName(name); - entityDefs.addType(entityDef, new AtlasEntityType(entityDef)); - } + registryData.enumDefs.addType(enumDef, new AtlasEnumType(enumDef)); + } else if (typeDef.getClass().equals(AtlasStructDef.class)) { + AtlasStructDef structDef = (AtlasStructDef) typeDef; - if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.updateTypeByNameWithNoRefResolve({})", name); - } - } + registryData.structDefs.addType(structDef, new AtlasStructType(structDef)); + } else if (typeDef.getClass().equals(AtlasClassificationDef.class)) { + AtlasClassificationDef classificationDef = (AtlasClassificationDef) typeDef; - public void removeTypeByGuid(String guid) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.removeTypeByGuid({})", guid); - } + registryData.classificationDefs.addType(classificationDef, + new AtlasClassificationType(classificationDef)); + } else if (typeDef.getClass().equals(AtlasEntityDef.class)) { + AtlasEntityDef entityDef = (AtlasEntityDef) typeDef; - if (guid != null) { - enumDefs.removeTypeDefByGuid(guid); - structDefs.removeTypeDefByGuid(guid); - classificationDefs.removeTypeDefByGuid(guid); - entityDefs.removeTypeDefByGuid(guid); - } + registryData.entityDefs.addType(entityDef, new AtlasEntityType(entityDef)); + } + } - if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.removeTypeByGuid({})", guid); + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasTypeRegistry.addTypeWithNoRefResolve({})", typeDef); + } } - } - public void removeTypeByName(String name) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.removeTypeByName({})", name); - } + private void addTypesWithNoRefResolve(Collection<? extends AtlasBaseTypeDef> typeDefs) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasTypeRegistry.addTypesWithNoRefResolve(length={})", + (typeDefs == null ? 0 : typeDefs.size())); + } - if (name != null) { - enumDefs.removeTypeDefByName(name); - structDefs.removeTypeDefByName(name); - classificationDefs.removeTypeDefByName(name); - entityDefs.removeTypeDefByName(name); - } + if (CollectionUtils.isNotEmpty(typeDefs)) { + for (AtlasBaseTypeDef typeDef : typeDefs) { + addTypeWithNoRefResolve(typeDef); + } + } - if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.removeEnumDefByName({})", name); + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasTypeRegistry.addTypesWithNoRefResolve(length={})", + (typeDefs == null ? 0 : typeDefs.size())); + } } - } - public void addTypes(Collection<? extends AtlasBaseTypeDef> typeDefs) throws AtlasBaseException { - if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.addTypes(length={})", (typeDefs == null ? 0 : typeDefs.size())); - } + private void updateTypeWithNoRefResolve(AtlasBaseTypeDef typeDef) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasTypeRegistry.updateType({})", typeDef); + } - if (CollectionUtils.isNotEmpty(typeDefs)) { - for (AtlasBaseTypeDef typeDef : typeDefs) { - addType(typeDef); + if (typeDef == null) { + // ignore + } else if (StringUtils.isNotBlank(typeDef.getGuid())) { + updateTypeByGuidWithNoRefResolve(typeDef.getGuid(), typeDef); + } else if (StringUtils.isNotBlank(typeDef.getName())) { + updateTypeByNameWithNoRefResolve(typeDef.getName(), typeDef); } - } - if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.addTypes(length={})", (typeDefs == null ? 0 : typeDefs.size())); + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasTypeRegistry.updateType({})", typeDef); + } } - } - public void addTypesWithNoRefResolve(Collection<? extends AtlasBaseTypeDef> typeDefs) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.addTypes(length={})", (typeDefs == null ? 0 : typeDefs.size())); - } + private void updateTypeByGuidWithNoRefResolve(String guid, AtlasBaseTypeDef typeDef) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasTypeRegistry.updateTypeByGuidWithNoRefResolve({})", guid); + } - if (CollectionUtils.isNotEmpty(typeDefs)) { - for (AtlasBaseTypeDef typeDef : typeDefs) { - addTypeWithNoRefResolve(typeDef); + if (guid == null || typeDef == null) { + // ignore + } else if (typeDef.getClass().equals(AtlasEnumDef.class)) { + AtlasEnumDef enumDef = (AtlasEnumDef)typeDef; + + registryData.enumDefs.removeTypeDefByGuid(guid); + registryData.enumDefs.addType(enumDef, new AtlasEnumType(enumDef)); + } else if (typeDef.getClass().equals(AtlasStructDef.class)) { + AtlasStructDef structDef = (AtlasStructDef)typeDef; + + registryData.structDefs.removeTypeDefByGuid(guid); + registryData.structDefs.addType(structDef, new AtlasStructType(structDef)); + } else if (typeDef.getClass().equals(AtlasClassificationDef.class)) { + AtlasClassificationDef classificationDef = (AtlasClassificationDef)typeDef; + + registryData.classificationDefs.removeTypeDefByGuid(guid); + registryData.classificationDefs.addType(classificationDef, + new AtlasClassificationType(classificationDef)); + } else if (typeDef.getClass().equals(AtlasEntityDef.class)) { + AtlasEntityDef entityDef = (AtlasEntityDef)typeDef; + + registryData.entityDefs.removeTypeDefByGuid(guid); + registryData.entityDefs.addType(entityDef, new AtlasEntityType(entityDef)); } - } - if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.addTypes(length={})", (typeDefs == null ? 0 : typeDefs.size())); + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasTypeRegistry.updateTypeByGuidWithNoRefResolve({})", guid); + } } - } - public void updateTypes(Collection<? extends AtlasBaseTypeDef> typeDefs) throws AtlasBaseException { - if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.updateTypes(length={})", (typeDefs == null ? 0 : typeDefs.size())); - } + private void updateTypeByNameWithNoRefResolve(String name, AtlasBaseTypeDef typeDef) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasTypeRegistry.updateTypeByNameWithNoRefResolve({})", name); + } - if (CollectionUtils.isNotEmpty(typeDefs)) { - for (AtlasBaseTypeDef typeDef : typeDefs) { - updateType(typeDef); + if (name == null || typeDef == null) { + // ignore + } else if (typeDef.getClass().equals(AtlasEnumDef.class)) { + AtlasEnumDef enumDef = (AtlasEnumDef)typeDef; + + registryData.enumDefs.removeTypeDefByName(name); + registryData.enumDefs.addType(enumDef, new AtlasEnumType(enumDef)); + } else if (typeDef.getClass().equals(AtlasStructDef.class)) { + AtlasStructDef structDef = (AtlasStructDef)typeDef; + + registryData.structDefs.removeTypeDefByName(name); + registryData.structDefs.addType(structDef, new AtlasStructType(structDef)); + } else if (typeDef.getClass().equals(AtlasClassificationDef.class)) { + AtlasClassificationDef classificationDef = (AtlasClassificationDef)typeDef; + + registryData.classificationDefs.removeTypeDefByName(name); + registryData.classificationDefs.addType(classificationDef, + new AtlasClassificationType(classificationDef)); + } else if (typeDef.getClass().equals(AtlasEntityDef.class)) { + AtlasEntityDef entityDef = (AtlasEntityDef)typeDef; + + registryData.entityDefs.removeTypeDefByName(name); + registryData.entityDefs.addType(entityDef, new AtlasEntityType(entityDef)); } - } - if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.updateTypes(length={})", (typeDefs == null ? 0 : typeDefs.size())); + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasTypeRegistry.updateTypeByNameWithNoRefResolve({})", name); + } } - } - public void updateTypesWithNoRefResolve(Collection<? extends AtlasBaseTypeDef> typeDefs) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.updateTypesWithNoRefResolve(length={})", (typeDefs == null ? 0 : typeDefs.size())); - } + private void updateTypesWithNoRefResolve(Collection<? extends AtlasBaseTypeDef> typeDefs) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasTypeRegistry.updateTypesWithNoRefResolve(length={})", + (typeDefs == null ? 0 : typeDefs.size())); + } - if (CollectionUtils.isNotEmpty(typeDefs)) { - for (AtlasBaseTypeDef typeDef : typeDefs) { - updateTypeWithNoRefResolve(typeDef); + if (CollectionUtils.isNotEmpty(typeDefs)) { + for (AtlasBaseTypeDef typeDef : typeDefs) { + updateTypeWithNoRefResolve(typeDef); + } } - } - if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.updateTypesWithNoRefResolve(length={})", (typeDefs == null ? 0 : typeDefs.size())); + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasTypeRegistry.updateTypesWithNoRefResolve(length={})", + (typeDefs == null ? 0 : typeDefs.size())); + } } } +} +class TypeCache { + private final Map<String, AtlasType> typeGuidMap; + private final Map<String, AtlasType> typeNameMap; - public Collection<AtlasEnumDef> getAllEnumDefs() { return enumDefs.getAll(); } - - public AtlasEnumDef getEnumDefByGuid(String guid) { - return enumDefs.getTypeDefByGuid(guid); + public TypeCache() { + typeGuidMap = new ConcurrentHashMap<>(); + typeNameMap = new ConcurrentHashMap<>(); } - public AtlasEnumDef getEnumDefByName(String name) { - return enumDefs.getTypeDefByName(name); + public TypeCache(TypeCache other) { + typeGuidMap = new ConcurrentHashMap<>(other.typeGuidMap); + typeNameMap = new ConcurrentHashMap<>(other.typeNameMap); } - - public Collection<AtlasStructDef> getAllStructDefs() { return structDefs.getAll(); } - - public AtlasStructDef getStructDefByGuid(String guid) { - return structDefs.getTypeDefByGuid(guid); + public void addType(AtlasType type) { + if (type != null) { + if (StringUtils.isNotEmpty(type.getTypeName())) { + typeNameMap.put(type.getTypeName(), type); + } + } } - public AtlasStructDef getStructDefByName(String name) { return structDefs.getTypeDefByName(name); } - - - public Collection<AtlasClassificationDef> getAllClassificationDefs() { return classificationDefs.getAll(); } + public void addType(AtlasBaseTypeDef typeDef, AtlasType type) { + if (typeDef != null && type != null) { + if (StringUtils.isNotEmpty(typeDef.getGuid())) { + typeGuidMap.put(typeDef.getGuid(), type); + } - public AtlasClassificationDef getClassificationDefByGuid(String guid) { - return classificationDefs.getTypeDefByGuid(guid); + if (StringUtils.isNotEmpty(typeDef.getName())) { + typeNameMap.put(typeDef.getName(), type); + } + } } - public AtlasClassificationDef getClassificationDefByName(String name) { - return classificationDefs.getTypeDefByName(name); + public Collection<String> getAllTypeNames() { + return Collections.unmodifiableCollection(typeNameMap.keySet()); } + public Collection<AtlasType> getAllTypes() { + return Collections.unmodifiableCollection(typeNameMap.values()); + } - public Collection<AtlasEntityDef> getAllEntityDefs() { return entityDefs.getAll(); } + public AtlasType getTypeByGuid(String guid) { + AtlasType ret = guid != null ? typeGuidMap.get(guid) : null; - public AtlasEntityDef getEntityDefByGuid(String guid) { - return entityDefs.getTypeDefByGuid(guid); + return ret; } - public AtlasEntityDef getEntityDefByName(String name) { - return entityDefs.getTypeDefByName(name); - } + public AtlasType getTypeByName(String name) { + AtlasType ret = name != null ? typeNameMap.get(name) : null; + return ret; + } - public void resolveReferences() throws AtlasBaseException { - for (Map.Entry<String, AtlasType> e : allTypes.entrySet()) { - e.getValue().resolveReferences(this); + public void removeTypeByGuid(String guid) { + if (guid != null) { + typeGuidMap.remove(guid); } } - - private void registerType(AtlasType dataType) { - allTypes.put(dataType.getTypeName(), dataType); + public void removeTypeByName(String name) { + if (name != null) { + typeNameMap.get(name); + } } +} - private void unregisterType(AtlasType dataType) { - allTypes.remove(dataType.getTypeName()); +class TypeDefCache<T extends AtlasBaseTypeDef> { + private final TypeCache typeCache; + private final Map<String, T> typeDefGuidMap; + private final Map<String, T> typeDefNameMap; + + public TypeDefCache(TypeCache typeCache) { + this.typeCache = typeCache; + this.typeDefGuidMap = new ConcurrentHashMap<>(); + this.typeDefNameMap = new ConcurrentHashMap<>(); } - private void unregisterTypeByName(String typeName) { - allTypes.remove(typeName); + public TypeDefCache(TypeDefCache other, TypeCache typeCache) { + this.typeCache = typeCache; + this.typeDefGuidMap = new ConcurrentHashMap<>(other.typeDefGuidMap); + this.typeDefNameMap = new ConcurrentHashMap<>(other.typeDefNameMap); } + public void addType(T typeDef, AtlasType type) { + if (typeDef != null && type != null) { + if (StringUtils.isNotEmpty(typeDef.getGuid())) { + typeDefGuidMap.put(typeDef.getGuid(), typeDef); + } - class TypeDefCache<T extends AtlasBaseTypeDef> { - private final AtlasTypeRegistry typeRegistry; - private final Map<String, T> typeDefGuidMap = new ConcurrentHashMap<String, T>(); - private final Map<String, T> typeDefNameMap = new ConcurrentHashMap<String, T>(); + if (StringUtils.isNotEmpty(typeDef.getName())) { + typeDefNameMap.put(typeDef.getName(), typeDef); + } - public TypeDefCache(AtlasTypeRegistry typeRegistry) { - this.typeRegistry = typeRegistry; + typeCache.addType(typeDef, type); } + } - public void addType(T typeDef, AtlasType type) { - if (type != null) { - if (StringUtils.isNotEmpty(typeDef.getGuid())) { - typeDefGuidMap.put(typeDef.getGuid(), typeDef); - } - - if (StringUtils.isNotEmpty(typeDef.getName())) { - typeDefNameMap.put(typeDef.getName(), typeDef); - } + public Collection<T> getAll() { + return Collections.unmodifiableCollection(typeDefNameMap.values()); + } - typeRegistry.registerType(type); - } - } + public T getTypeDefByGuid(String guid) { + T ret = guid != null ? typeDefGuidMap.get(guid) : null; - public Collection<T> getAll() { - return Collections.unmodifiableCollection(typeDefNameMap.values()); - } + return ret; + } - public T getTypeDefByGuid(String guid) { - T ret = guid != null ? typeDefGuidMap.get(guid) : null; + public T getTypeDefByName(String name) { + T ret = name != null ? typeDefNameMap.get(name) : null; - return ret; - } + return ret; + } - public T getTypeDefByName(String name) { - T ret = name != null ? typeDefNameMap.get(name) : null; + public void removeTypeDefByGuid(String guid) { + if (guid != null) { + T typeDef = typeDefGuidMap.remove(guid); - return ret; - } + typeCache.removeTypeByGuid(guid); - public void removeTypeDefByGuid(String guid) { - T typeDef = guid != null ? typeDefGuidMap.remove(guid) : null; - String name = typeDef != null ? typeDef.getName() : null; + String name = typeDef != null ? typeDef.getName() : null; if (name != null) { typeDefNameMap.remove(name); - typeRegistry.unregisterTypeByName(name); + typeCache.removeTypeByName(name); } + } + } + + public void removeTypeDefByName(String name) { + if (name != null) { + T typeDef = typeDefNameMap.remove(name); + + typeCache.removeTypeByName(name); - public void removeTypeDefByName(String name) { - T typeDef = name != null ? typeDefNameMap.get(name) : null; - String guid = typeDef != null ? typeDef.getGuid() : null; + String guid = typeDef != null ? typeDef.getGuid() : null; if (guid != null) { typeDefGuidMap.remove(guid); - } - - if (name != null) { - typeRegistry.unregisterTypeByName(name); + typeCache.removeTypeByGuid(guid); } } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/eb6e656b/intg/src/test/java/org/apache/atlas/model/ModelTestUtil.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/model/ModelTestUtil.java b/intg/src/test/java/org/apache/atlas/model/ModelTestUtil.java index e2db68e..c0bb1f2 100644 --- a/intg/src/test/java/org/apache/atlas/model/ModelTestUtil.java +++ b/intg/src/test/java/org/apache/atlas/model/ModelTestUtil.java @@ -41,6 +41,7 @@ import org.apache.atlas.type.AtlasClassificationType; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.type.AtlasTypeRegistry.AtlasTransientTypeRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -158,7 +159,11 @@ public final class ModelTestUtil { } try { - typesRegistry.addType(ret); + AtlasTransientTypeRegistry ttr = typesRegistry.createTransientTypeRegistry(); + + ttr.addType(ret); + + typesRegistry.commitTransientTypeRegistry(ttr); } catch (AtlasBaseException excp) { LOG.error("failed to create enum-def", excp); @@ -182,7 +187,11 @@ public final class ModelTestUtil { ret.setAttributeDefs(newAttributeDefsWithAllBuiltInTypes(PREFIX_ATTRIBUTE_NAME)); try { - typesRegistry.addType(ret); + AtlasTransientTypeRegistry ttr = typesRegistry.createTransientTypeRegistry(); + + ttr.addType(ret); + + typesRegistry.commitTransientTypeRegistry(ttr); } catch (AtlasBaseException excp) { LOG.error("failed to create struct-def", excp); @@ -220,7 +229,11 @@ public final class ModelTestUtil { } try { - typesRegistry.addType(ret); + AtlasTransientTypeRegistry ttr = typesRegistry.createTransientTypeRegistry(); + + ttr.addType(ret); + + typesRegistry.commitTransientTypeRegistry(ttr); } catch (AtlasBaseException excp) { LOG.error("failed to create entity-def", excp); @@ -267,7 +280,11 @@ public final class ModelTestUtil { } try { - typesRegistry.addType(ret); + AtlasTransientTypeRegistry ttr = typesRegistry.createTransientTypeRegistry(); + + ttr.addType(ret); + + typesRegistry.commitTransientTypeRegistry(ttr); } catch (AtlasBaseException excp) { LOG.error("failed to create classification-def", excp); @@ -285,10 +302,14 @@ public final class ModelTestUtil { public static AtlasEntity newEntity(AtlasEntityDef entityDef, AtlasTypeRegistry typesRegistry) { AtlasEntity ret = null; - AtlasType dataType = typesRegistry.getType(entityDef.getName()); + try { + AtlasType dataType = typesRegistry.getType(entityDef.getName()); - if (dataType != null && dataType instanceof AtlasEntityType) { - ret = ((AtlasEntityType)dataType).createDefaultValue(); + if (dataType instanceof AtlasEntityType) { + ret = ((AtlasEntityType) dataType).createDefaultValue(); + } + } catch (AtlasBaseException excp) { + LOG.error("failed to get entity-type " + entityDef.getName(), excp); } return ret; @@ -301,10 +322,14 @@ public final class ModelTestUtil { public static AtlasStruct newStruct(AtlasStructDef structDef, AtlasTypeRegistry typesRegistry) { AtlasStruct ret = null; - AtlasType dataType = typesRegistry.getType(structDef.getName()); + try { + AtlasType dataType = typesRegistry.getType(structDef.getName()); - if (dataType != null && dataType instanceof AtlasStructType) { - ret = ((AtlasStructType)dataType).createDefaultValue(); + if (dataType instanceof AtlasStructType) { + ret = ((AtlasStructType)dataType).createDefaultValue(); + } + } catch (AtlasBaseException excp) { + LOG.error("failed to get struct-type " + structDef.getName(), excp); } return ret; @@ -318,10 +343,14 @@ public final class ModelTestUtil { AtlasTypeRegistry typesRegistry) { AtlasClassification ret = null; - AtlasType dataType = typesRegistry.getType(classificationDef.getName()); + try { + AtlasType dataType = typesRegistry.getType(classificationDef.getName()); - if (dataType != null && dataType instanceof AtlasClassificationType) { - ret = ((AtlasClassificationType)dataType).createDefaultValue(); + if (dataType instanceof AtlasClassificationType) { + ret = ((AtlasClassificationType)dataType).createDefaultValue(); + } + } catch (AtlasBaseException excp) { + LOG.error("failed to get classification-type " + classificationDef.getName(), excp); } return ret; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/eb6e656b/intg/src/test/java/org/apache/atlas/model/instance/TestAtlasClassification.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/model/instance/TestAtlasClassification.java b/intg/src/test/java/org/apache/atlas/model/instance/TestAtlasClassification.java index 577dccf..6dc72ee 100644 --- a/intg/src/test/java/org/apache/atlas/model/instance/TestAtlasClassification.java +++ b/intg/src/test/java/org/apache/atlas/model/instance/TestAtlasClassification.java @@ -17,6 +17,7 @@ */ package org.apache.atlas.model.instance; +import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.ModelTestUtil; import org.apache.atlas.model.typedef.AtlasClassificationDef; import org.apache.atlas.type.AtlasType; @@ -31,7 +32,7 @@ import static org.testng.Assert.assertTrue; public class TestAtlasClassification { @Test - public void testClassificationSerDe() { + public void testClassificationSerDe() throws AtlasBaseException { AtlasClassificationDef classificationDef = ModelTestUtil.getClassificationDef(); AtlasTypeRegistry typeRegistry = ModelTestUtil.getTypesRegistry(); AtlasType dataType = typeRegistry.getType(classificationDef.getName()); @@ -50,7 +51,7 @@ public class TestAtlasClassification { } @Test - public void testClassificationSerDeWithSuperType() { + public void testClassificationSerDeWithSuperType() throws AtlasBaseException { AtlasClassificationDef classificationDef = ModelTestUtil.getClassificationDefWithSuperType(); AtlasTypeRegistry typeRegistry = ModelTestUtil.getTypesRegistry(); AtlasType dataType = typeRegistry.getType(classificationDef.getName()); @@ -69,7 +70,7 @@ public class TestAtlasClassification { } @Test - public void testClassificationSerDeWithSuperTypes() { + public void testClassificationSerDeWithSuperTypes() throws AtlasBaseException { AtlasClassificationDef classificationDef = ModelTestUtil.getClassificationDefWithSuperTypes(); AtlasTypeRegistry typeRegistry = ModelTestUtil.getTypesRegistry(); AtlasType dataType = typeRegistry.getType(classificationDef.getName()); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/eb6e656b/intg/src/test/java/org/apache/atlas/model/instance/TestAtlasEntity.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/model/instance/TestAtlasEntity.java b/intg/src/test/java/org/apache/atlas/model/instance/TestAtlasEntity.java index fbf1cc7..efcf1cf 100644 --- a/intg/src/test/java/org/apache/atlas/model/instance/TestAtlasEntity.java +++ b/intg/src/test/java/org/apache/atlas/model/instance/TestAtlasEntity.java @@ -17,6 +17,7 @@ */ package org.apache.atlas.model.instance; +import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.ModelTestUtil; import org.apache.atlas.model.typedef.AtlasEntityDef; import org.apache.atlas.type.AtlasType; @@ -31,7 +32,7 @@ import static org.testng.Assert.assertTrue; public class TestAtlasEntity { @Test - public void testEntitySerDe() { + public void testEntitySerDe() throws AtlasBaseException { AtlasEntityDef entityDef = ModelTestUtil.getEntityDef(); AtlasTypeRegistry typeRegistry = ModelTestUtil.getTypesRegistry(); AtlasType dataType = typeRegistry.getType(entityDef.getName()); @@ -50,7 +51,7 @@ public class TestAtlasEntity { } @Test - public void testEntitySerDeWithSuperType() { + public void testEntitySerDeWithSuperType() throws AtlasBaseException { AtlasEntityDef entityDef = ModelTestUtil.getEntityDefWithSuperType(); AtlasTypeRegistry typeRegistry = ModelTestUtil.getTypesRegistry(); AtlasType dataType = typeRegistry.getType(entityDef.getName()); @@ -69,7 +70,7 @@ public class TestAtlasEntity { } @Test - public void testEntitySerDeWithSuperTypes() { + public void testEntitySerDeWithSuperTypes() throws AtlasBaseException { AtlasEntityDef entityDef = ModelTestUtil.getEntityDefWithSuperTypes(); AtlasTypeRegistry typeRegistry = ModelTestUtil.getTypesRegistry(); AtlasType dataType = typeRegistry.getType(entityDef.getName()); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/eb6e656b/intg/src/test/java/org/apache/atlas/model/typedef/TestAtlasEntityDef.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/model/typedef/TestAtlasEntityDef.java b/intg/src/test/java/org/apache/atlas/model/typedef/TestAtlasEntityDef.java index f3b12ef..b8cc77c 100644 --- a/intg/src/test/java/org/apache/atlas/model/typedef/TestAtlasEntityDef.java +++ b/intg/src/test/java/org/apache/atlas/model/typedef/TestAtlasEntityDef.java @@ -84,10 +84,12 @@ public class TestAtlasEntityDef { entityDef.addSuperType(newSuperType); assertTrue(entityDef.hasSuperType(newSuperType)); + + entityDef.removeSuperType(newSuperType); } @Test - public void testEntityDefDefRemoveElement() { + public void testEntityDefRemoveElement() { AtlasEntityDef entityDef = ModelTestUtil.newEntityDefWithSuperTypes(); for (String superType : entityDef.getSuperTypes()) { @@ -114,6 +116,9 @@ public class TestAtlasEntityDef { for (String superType : newSuperTypes) { assertTrue(entityDef.hasSuperType(superType)); } + + // restore old sypertypes + entityDef.setSuperTypes(oldSuperTypes); } @Test http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/eb6e656b/intg/src/test/java/org/apache/atlas/type/TestAtlasEntityType.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/type/TestAtlasEntityType.java b/intg/src/test/java/org/apache/atlas/type/TestAtlasEntityType.java index ec893a0..90fea9f 100644 --- a/intg/src/test/java/org/apache/atlas/type/TestAtlasEntityType.java +++ b/intg/src/test/java/org/apache/atlas/type/TestAtlasEntityType.java @@ -17,7 +17,11 @@ */ package org.apache.atlas.type; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.ModelTestUtil; @@ -26,6 +30,7 @@ import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import org.apache.atlas.model.typedef.AtlasEntityDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef; +import org.apache.atlas.type.AtlasTypeRegistry.AtlasTransientTypeRegistry; import org.testng.annotations.Test; import static org.testng.Assert.*; @@ -127,8 +132,11 @@ public class TestAtlasEntityType { entityDefs.add(createColumnEntityDef()); try { - typeRegistry.addTypesWithNoRefResolve(entityDefs); - typeRegistry.resolveReferences(); + AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry(); + + ttr.addTypes(entityDefs); + + typeRegistry.commitTransientTypeRegistry(ttr); } catch (AtlasBaseException excp) { failureMsg = excp.getMessage(); } @@ -144,7 +152,11 @@ public class TestAtlasEntityType { entityDefs.add(createTableEntityDef()); try { - typeRegistry.addTypes(entityDefs); + AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry(); + + ttr.addTypes(entityDefs); + + typeRegistry.commitTransientTypeRegistry(ttr); } catch (AtlasBaseException excp) { failureMsg = excp.getMessage(); } @@ -161,8 +173,11 @@ public class TestAtlasEntityType { entityDefs.add(createColumnEntityDef()); try { - typeRegistry.addTypesWithNoRefResolve(entityDefs); - typeRegistry.resolveReferences(); + AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry(); + + ttr.addTypes(entityDefs); + + typeRegistry.commitTransientTypeRegistry(ttr); } catch (AtlasBaseException excp) { failureMsg = excp.getMessage(); } @@ -178,7 +193,11 @@ public class TestAtlasEntityType { entityDefs.add(createColumnEntityDef()); try { - typeRegistry.addTypes(entityDefs); + AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry(); + + ttr.addTypes(entityDefs); + + typeRegistry.commitTransientTypeRegistry(ttr); } catch (AtlasBaseException excp) { failureMsg = excp.getMessage(); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/eb6e656b/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 0792d46..d2b848b 100644 --- a/release-log.txt +++ b/release-log.txt @@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai) ALL CHANGES: +ATLAS-1230 updated AtlasTypeRegistry to support batch, atomic type updates (mneethiraj) ATLAS-1229 Add TypeCategory and methods to access attribute definitiions in AtlasTypes (sumasai) ATLAS-1227 Added support for attribute constraints in the API (mneethiraj) ATLAS-1225 Optimize AtlasTypeDefGraphStore to use typeRegistry (mneethiraj via sumasai)
