Repository: incubator-atlas Updated Branches: refs/heads/master 51d24ae3b -> e5fe91a44
ATLAS-1225: Updated AtlasTypeDefGraphStore to use AtlasTypesRegistry, to reduce query to Graph DB Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/e5fe91a4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/e5fe91a4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/e5fe91a4 Branch: refs/heads/master Commit: e5fe91a44bff992e571d3ff5811113068d635e36 Parents: 51d24ae Author: Madhan Neethiraj <[email protected]> Authored: Thu Oct 13 13:57:03 2016 -0700 Committer: Suma Shivaprasad <[email protected]> Committed: Mon Oct 17 11:54:30 2016 -0700 ---------------------------------------------------------------------- .../atlas/model/typedef/AtlasStructDef.java | 14 +- .../apache/atlas/store/AtlasTypeDefStore.java | 2 +- .../org/apache/atlas/type/AtlasArrayType.java | 16 + .../org/apache/atlas/type/AtlasEnumType.java | 5 + .../org/apache/atlas/type/AtlasMapType.java | 20 + .../org/apache/atlas/type/AtlasStructType.java | 31 +- .../java/org/apache/atlas/type/AtlasType.java | 1 - .../apache/atlas/type/AtlasTypeRegistry.java | 405 ++++++++++++------- .../org/apache/atlas/model/ModelTestUtil.java | 14 +- .../apache/atlas/type/TestAtlasEntityType.java | 2 +- release-log.txt | 1 + .../store/graph/AtlasTypeDefGraphStore.java | 356 ++++++++++------ .../graph/v1/AtlasClassificationDefStoreV1.java | 2 - .../store/graph/v1/AtlasStructDefStoreV1.java | 28 +- .../graph/v1/AtlasTypeDefGraphStoreV1.java | 24 +- .../org/apache/atlas/web/rest/TypesREST.java | 2 +- 16 files changed, 609 insertions(+), 314 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e5fe91a4/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java index ebfd535..55bff2b 100644 --- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java +++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java @@ -195,7 +195,16 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { sb.append("AtlasStructDef{"); super.toString(sb); sb.append(", attributeDefs=["); - dumpObjects(attributeDefs, sb); + if (CollectionUtils.isNotEmpty(attributeDefs)) { + int i = 0; + for (AtlasAttributeDef attributeDef : attributeDefs) { + attributeDef.toString(sb); + if (i > 0) { + sb.append(", "); + } + i++; + } + } sb.append("]"); sb.append('}'); @@ -287,7 +296,6 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { } } - public String getName() { return name; } @@ -350,7 +358,6 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { isIndexable = idexable; } - public StringBuilder toString(StringBuilder sb) { if (sb == null) { sb = new StringBuilder(); @@ -410,7 +417,6 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { } } - /** * REST serialization friendly list. */ http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e5fe91a4/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java b/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java index bc95288..058ea44 100644 --- a/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java +++ b/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java @@ -114,7 +114,7 @@ public interface AtlasTypeDefStore { /*************************/ /** EntityDef operation **/ /*************************/ - AtlasEntityDef createEntityDefs(AtlasEntityDef entityDef) throws AtlasBaseException; + AtlasEntityDef createEntityDef(AtlasEntityDef entityDef) throws AtlasBaseException; List<AtlasEntityDef> createEntityDefs(List<AtlasEntityDef> entityDefs) throws AtlasBaseException; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e5fe91a4/intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java b/intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java index 1473c3a..2b79974 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java @@ -84,6 +84,22 @@ public class AtlasArrayType extends AtlasType { this.resolveReferences(typeRegistry); } + public String getElementTypeName() { + return elementTypeName; + } + + public int getMinCount() { + return minCount; + } + + public int getMaxCount() { + return maxCount; + } + + public AtlasType getElementType() { + return elementType; + } + @Override public void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { elementType = typeRegistry.getType(elementTypeName); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e5fe91a4/intg/src/main/java/org/apache/atlas/type/AtlasEnumType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEnumType.java b/intg/src/main/java/org/apache/atlas/type/AtlasEnumType.java index 34f5d49..5b41e71 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasEnumType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasEnumType.java @@ -22,6 +22,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.typedef.AtlasEnumDef; import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumElementDef; @@ -59,6 +60,10 @@ public class AtlasEnumType extends AtlasType { } @Override + public void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { + } + + @Override public Object createDefaultValue() { return defaultValue; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e5fe91a4/intg/src/main/java/org/apache/atlas/type/AtlasMapType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasMapType.java b/intg/src/main/java/org/apache/atlas/type/AtlasMapType.java index ca5bb21..d770bc7 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasMapType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasMapType.java @@ -64,6 +64,26 @@ public class AtlasMapType extends AtlasType { resolveReferences(typeRegistry); } + public String getKeyTypeName() { + return keyTypeName; + } + + public String getValueTypeName() { + return valueTypeName; + } + + public AtlasType getKeyType() { + return keyType; + } + + public AtlasType getValueType() { + return valueType; + } + + public void setKeyType(AtlasType keyType) { + this.keyType = keyType; + } + @Override public void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { this.keyType = typeRegistry.getType(keyTypeName); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e5fe91a4/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java index 096f061..8813542 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java @@ -38,6 +38,7 @@ public class AtlasStructType extends AtlasType { private Map<String, AtlasType> attrTypes = Collections.emptyMap(); + public AtlasStructType(AtlasStructDef structDef) { super(structDef.getName()); @@ -52,6 +53,10 @@ public class AtlasStructType extends AtlasType { this.resolveReferences(typeRegistry); } + public AtlasType getAttributeType(String attributeName) { return attrTypes.get(attributeName); } + + public AtlasAttributeDef getAttributeDef(String attributeName) { return structDef.getAttribute(attributeName); } + @Override public void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { Map<String, AtlasType> a = new HashMap<String, AtlasType>(); @@ -59,24 +64,20 @@ public class AtlasStructType extends AtlasType { for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) { AtlasType attrType = typeRegistry.getType(attributeDef.getTypeName()); - if (attrType != null) { - Cardinality cardinality = attributeDef.getCardinality(); - - if (cardinality == Cardinality.LIST || cardinality == Cardinality.SET) { - attrType = new AtlasArrayType(attrType, - attributeDef.getValuesMinCount(), - attributeDef.getValuesMaxCount()); - } - - a.put(attributeDef.getName(), attrType); - } else { - String msg = attributeDef.getTypeName() + ": unknown type for attribute " - + structDef.getName() + "." + attributeDef.getName(); + if (attrType == null) { + throw new AtlasBaseException(attributeDef.getTypeName() + ": unknown type for attribute " + + structDef.getName() + "." + attributeDef.getName()); + } - LOG.error(msg); + Cardinality cardinality = attributeDef.getCardinality(); - throw new AtlasBaseException(msg); + if (cardinality == Cardinality.LIST || cardinality == Cardinality.SET) { + attrType = new AtlasArrayType(attrType, + attributeDef.getValuesMinCount(), + attributeDef.getValuesMaxCount()); } + + a.put(attributeDef.getName(), attrType); } this.attrTypes = Collections.unmodifiableMap(a); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e5fe91a4/intg/src/main/java/org/apache/atlas/type/AtlasType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasType.java b/intg/src/main/java/org/apache/atlas/type/AtlasType.java index a3f1b02..204e305 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasType.java @@ -41,7 +41,6 @@ public abstract class AtlasType { } public void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { - } public String getTypeName() { return typeName; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e5fe91a4/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java index f4683ff..c3f17f4 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java @@ -18,12 +18,18 @@ package org.apache.atlas.type; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.typedef.*; +import org.apache.atlas.model.typedef.AtlasBaseTypeDef; +import org.apache.atlas.model.typedef.AtlasClassificationDef; +import org.apache.atlas.model.typedef.AtlasEnumDef; +import org.apache.atlas.model.typedef.AtlasEntityDef; +import org.apache.atlas.model.typedef.AtlasStructDef; import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_ARRAY_PREFIX; import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_ARRAY_SUFFIX; import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_MAP_PREFIX; import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_MAP_SUFFIX; import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_MAP_KEY_VAL_SEP; + +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,7 +51,6 @@ public class AtlasTypeRegistry { private final TypeDefCache<AtlasClassificationDef> classificationDefs; private final TypeDefCache<AtlasEntityDef> entityDefs; - public AtlasTypeRegistry() { allTypes = new ConcurrentHashMap<>(); enumDefs = new TypeDefCache<>(this); @@ -67,12 +72,6 @@ public class AtlasTypeRegistry { registerType(new AtlasBuiltInTypes.AtlasObjectIdType()); } - public void resolveReferences() throws AtlasBaseException { - for (Map.Entry<String, AtlasType> e : allTypes.entrySet()) { - e.getValue().resolveReferences(this); - } - } - public Collection<String> getAllTypeNames() { return Collections.unmodifiableSet(allTypes.keySet()); } public AtlasType getType(String typeName) { @@ -112,260 +111,386 @@ public class AtlasTypeRegistry { } - public void addEnumDef(AtlasEnumDef enumDef) { + public void addType(AtlasBaseTypeDef typeDef) throws AtlasBaseException { if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.addEnumDef({})", enumDef); + LOG.debug("==> AtlasTypeRegistry.addType({})", typeDef); } - enumDefs.addType(enumDef, new AtlasEnumType(enumDef)); + if (typeDef == null) { + // ignore + } else if (typeDef.getClass().equals(AtlasEnumDef.class)) { + AtlasEnumDef enumDef = (AtlasEnumDef)typeDef; - if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.addEnumDef({})", enumDef); - } - } + enumDefs.addType(enumDef, new AtlasEnumType(enumDef)); + } else if (typeDef.getClass().equals(AtlasStructDef.class)) { + AtlasStructDef structDef = (AtlasStructDef)typeDef; - public Collection<AtlasEnumDef> getAllEnumDefs() { return enumDefs.getAll(); } + structDefs.addType(structDef, new AtlasStructType(structDef, this)); + } else if (typeDef.getClass().equals(AtlasClassificationDef.class)) { + AtlasClassificationDef classificationDef = (AtlasClassificationDef)typeDef; - public AtlasEnumDef getEnumDefByGuid(String guid) { - return enumDefs.getTypeDefByGuid(guid); - } + classificationDefs.addType(classificationDef, new AtlasClassificationType(classificationDef, this)); + } else if (typeDef.getClass().equals(AtlasEntityDef.class)) { + AtlasEntityDef entityDef = (AtlasEntityDef)typeDef; - public AtlasEnumDef getEnumDefByName(String name) { - return enumDefs.getTypeDefByName(name); + entityDefs.addType(entityDef, new AtlasEntityType(entityDef, this)); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasTypeRegistry.addType({})", typeDef); + } } - public void removeEnumDefByGuid(String guid) { + public void addTypeWithNoRefResolve(AtlasBaseTypeDef typeDef) { if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.removeEnumDefByGuid({})", guid); + LOG.debug("==> AtlasTypeRegistry.addTypeWithNoRefResolve({})", typeDef); } - AtlasEnumDef enumDef = enumDefs.getTypeDefByGuid(guid); + if (typeDef == null) { + // ignore + } else if (typeDef.getClass().equals(AtlasEnumDef.class)) { + AtlasEnumDef enumDef = (AtlasEnumDef)typeDef; - if (enumDef != null) { - enumDefs.removeTypeDefByGuid(guid); + enumDefs.addType(enumDef, new AtlasEnumType(enumDef)); + } else if (typeDef.getClass().equals(AtlasStructDef.class)) { + AtlasStructDef structDef = (AtlasStructDef)typeDef; + + structDefs.addType(structDef, new AtlasStructType(structDef)); + } else if (typeDef.getClass().equals(AtlasClassificationDef.class)) { + AtlasClassificationDef classificationDef = (AtlasClassificationDef)typeDef; + + classificationDefs.addType(classificationDef, new AtlasClassificationType(classificationDef)); + } else if (typeDef.getClass().equals(AtlasEntityDef.class)) { + AtlasEntityDef entityDef = (AtlasEntityDef)typeDef; + + entityDefs.addType(entityDef, new AtlasEntityType(entityDef)); } if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.removeEnumDefByGuid({})", guid); + LOG.debug("<== AtlasTypeRegistry.addTypeWithNoRefResolve({})", typeDef); } } - public void removeEnumDefByName(String name) { + public void updateType(AtlasBaseTypeDef typeDef) throws AtlasBaseException { if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.removeEnumDefByName({})", name); + LOG.debug("==> AtlasTypeRegistry.updateType({})", typeDef); } - AtlasEnumDef enumDef = enumDefs.getTypeDefByName(name); - - if (enumDef != null) { - enumDefs.removeTypeDefByName(name); + if (typeDef == null) { + // ignore + } else if (StringUtils.isNotBlank(typeDef.getGuid())) { + updateTypeByGuid(typeDef.getGuid(), typeDef); + } else if (StringUtils.isNotBlank(typeDef.getName())) { + updateTypeByName(typeDef.getName(), typeDef); } if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.removeEnumDefByName({})", name); + LOG.debug("<== AtlasTypeRegistry.updateType({})", typeDef); } } - - public void addStructDefWithNoRefResolve(AtlasStructDef structDef) { + public void updateTypeWithNoRefResolve(AtlasBaseTypeDef typeDef) { if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.addStructDefWithNoRefResolve({})", structDef); + LOG.debug("==> AtlasTypeRegistry.updateType({})", typeDef); } - structDefs.addType(structDef, new AtlasStructType(structDef)); + if (typeDef == null) { + // ignore + } else if (StringUtils.isNotBlank(typeDef.getGuid())) { + updateTypeByGuidWithNoRefResolve(typeDef.getGuid(), typeDef); + } else if (StringUtils.isNotBlank(typeDef.getName())) { + updateTypeByNameWithNoRefResolve(typeDef.getName(), typeDef); + } if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.addStructDefWithNoRefResolve({})", structDef); + LOG.debug("<== AtlasTypeRegistry.updateType({})", typeDef); } } - public void addStructDef(AtlasStructDef structDef) throws AtlasBaseException { + public void updateTypeByGuid(String guid, AtlasBaseTypeDef typeDef) throws AtlasBaseException { if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.addStructDef({})", structDef); + LOG.debug("==> AtlasTypeRegistry.updateTypeByGuid({})", guid); } - structDefs.addType(structDef, new AtlasStructType(structDef, this)); + if (guid == null || typeDef == null) { + // ignore + } else if (typeDef.getClass().equals(AtlasEnumDef.class)) { + AtlasEnumDef enumDef = (AtlasEnumDef)typeDef; + + enumDefs.removeTypeDefByGuid(guid); + enumDefs.addType(enumDef, new AtlasEnumType(enumDef)); + } else if (typeDef.getClass().equals(AtlasStructDef.class)) { + AtlasStructDef structDef = (AtlasStructDef)typeDef; + + structDefs.removeTypeDefByGuid(guid); + structDefs.addType(structDef, new AtlasStructType(structDef, this)); + } else if (typeDef.getClass().equals(AtlasClassificationDef.class)) { + AtlasClassificationDef classificationDef = (AtlasClassificationDef)typeDef; + + classificationDefs.removeTypeDefByGuid(guid); + classificationDefs.addType(classificationDef, new AtlasClassificationType(classificationDef, this)); + } else if (typeDef.getClass().equals(AtlasEntityDef.class)) { + AtlasEntityDef entityDef = (AtlasEntityDef)typeDef; + + entityDefs.removeTypeDefByGuid(guid); + entityDefs.addType(entityDef, new AtlasEntityType(entityDef, this)); + } if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.addStructDef({})", structDef); + LOG.debug("<== AtlasTypeRegistry.updateTypeByGuid({})", guid); } } - public Collection<AtlasStructDef> getAllStructDefs() { return structDefs.getAll(); } + public void updateTypeByName(String name, AtlasBaseTypeDef typeDef) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasTypeRegistry.updateEnumDefByName({})", name); + } - public AtlasStructDef getStructDefByGuid(String guid) { - return structDefs.getTypeDefByGuid(guid); - } + if (name == null || typeDef == null) { + // ignore + } else if (typeDef.getClass().equals(AtlasEnumDef.class)) { + AtlasEnumDef enumDef = (AtlasEnumDef)typeDef; - public AtlasStructDef getStructDefByName(String name) { return structDefs.getTypeDefByName(name); } + enumDefs.removeTypeDefByName(name); + enumDefs.addType(enumDef, new AtlasEnumType(enumDef)); + } else if (typeDef.getClass().equals(AtlasStructDef.class)) { + AtlasStructDef structDef = (AtlasStructDef)typeDef; - public void removeStructDefByGuid(String guid) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.removeStructDefByGuid({})", guid); - } + structDefs.removeTypeDefByName(name); + structDefs.addType(structDef, new AtlasStructType(structDef, this)); + } else if (typeDef.getClass().equals(AtlasClassificationDef.class)) { + AtlasClassificationDef classificationDef = (AtlasClassificationDef)typeDef; - AtlasStructDef structDef = structDefs.getTypeDefByGuid(guid); + classificationDefs.removeTypeDefByName(name); + classificationDefs.addType(classificationDef, new AtlasClassificationType(classificationDef, this)); + } else if (typeDef.getClass().equals(AtlasEntityDef.class)) { + AtlasEntityDef entityDef = (AtlasEntityDef)typeDef; - if (structDef != null) { - structDefs.removeTypeDefByGuid(guid); + entityDefs.removeTypeDefByName(name); + entityDefs.addType(entityDef, new AtlasEntityType(entityDef, this)); } if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.removeStructDefByGuid({})", guid); + LOG.debug("<== AtlasTypeRegistry.updateEnumDefByName({})", name); } } - public void removeStructDefByName(String name) { + public void updateTypeByGuidWithNoRefResolve(String guid, AtlasBaseTypeDef typeDef) { if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.removeStructDefByName({})", name); + LOG.debug("==> AtlasTypeRegistry.updateTypeByGuidWithNoRefResolve({})", guid); } - AtlasStructDef structDef = structDefs.getTypeDefByName(name); + if (guid == null || typeDef == null) { + // ignore + } else if (typeDef.getClass().equals(AtlasEnumDef.class)) { + AtlasEnumDef enumDef = (AtlasEnumDef)typeDef; - if (structDef != null) { - structDefs.removeTypeDefByName(name); + enumDefs.removeTypeDefByGuid(guid); + enumDefs.addType(enumDef, new AtlasEnumType(enumDef)); + } else if (typeDef.getClass().equals(AtlasStructDef.class)) { + AtlasStructDef structDef = (AtlasStructDef)typeDef; + + structDefs.removeTypeDefByGuid(guid); + structDefs.addType(structDef, new AtlasStructType(structDef)); + } else if (typeDef.getClass().equals(AtlasClassificationDef.class)) { + AtlasClassificationDef classificationDef = (AtlasClassificationDef)typeDef; + + classificationDefs.removeTypeDefByGuid(guid); + classificationDefs.addType(classificationDef, new AtlasClassificationType(classificationDef)); + } else if (typeDef.getClass().equals(AtlasEntityDef.class)) { + AtlasEntityDef entityDef = (AtlasEntityDef)typeDef; + + entityDefs.removeTypeDefByGuid(guid); + entityDefs.addType(entityDef, new AtlasEntityType(entityDef)); } if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.removeStructDefByName({})", name); + LOG.debug("<== AtlasTypeRegistry.updateTypeByGuidWithNoRefResolve({})", guid); } } - - public void addClassificationDefWithNoRefResolve(AtlasClassificationDef classificationDef) { + public void updateTypeByNameWithNoRefResolve(String name, AtlasBaseTypeDef typeDef) { if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.addClassificationDefWithNoRefResolve({})", classificationDef); + LOG.debug("==> AtlasTypeRegistry.updateTypeByNameWithNoRefResolve({})", name); } - classificationDefs.addType(classificationDef, new AtlasClassificationType(classificationDef)); + if (name == null || typeDef == null) { + // ignore + } else if (typeDef.getClass().equals(AtlasEnumDef.class)) { + AtlasEnumDef enumDef = (AtlasEnumDef)typeDef; + + enumDefs.removeTypeDefByName(name); + enumDefs.addType(enumDef, new AtlasEnumType(enumDef)); + } else if (typeDef.getClass().equals(AtlasStructDef.class)) { + AtlasStructDef structDef = (AtlasStructDef)typeDef; + + structDefs.removeTypeDefByName(name); + structDefs.addType(structDef, new AtlasStructType(structDef)); + } else if (typeDef.getClass().equals(AtlasClassificationDef.class)) { + AtlasClassificationDef classificationDef = (AtlasClassificationDef)typeDef; + + classificationDefs.removeTypeDefByName(name); + classificationDefs.addType(classificationDef, new AtlasClassificationType(classificationDef)); + } else if (typeDef.getClass().equals(AtlasEntityDef.class)) { + AtlasEntityDef entityDef = (AtlasEntityDef)typeDef; + + entityDefs.removeTypeDefByName(name); + entityDefs.addType(entityDef, new AtlasEntityType(entityDef)); + } if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.addClassificationDefWithNoRefResolve({})", classificationDef); + LOG.debug("<== AtlasTypeRegistry.updateTypeByNameWithNoRefResolve({})", name); } } - public void addClassificationDef(AtlasClassificationDef classificationDef) - throws AtlasBaseException { + public void removeTypeByGuid(String guid) { if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.addClassificationDef({})", classificationDef); + LOG.debug("==> AtlasTypeRegistry.removeTypeByGuid({})", guid); } - classificationDefs.addType(classificationDef, new AtlasClassificationType(classificationDef, this)); + if (guid != null) { + enumDefs.removeTypeDefByGuid(guid); + structDefs.removeTypeDefByGuid(guid); + classificationDefs.removeTypeDefByGuid(guid); + entityDefs.removeTypeDefByGuid(guid); + } if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.addClassificationDef({})", classificationDef); + LOG.debug("<== AtlasTypeRegistry.removeTypeByGuid({})", guid); } } - public Collection<AtlasClassificationDef> getAllClassificationDefs() { return classificationDefs.getAll(); } + public void removeTypeByName(String name) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasTypeRegistry.removeTypeByName({})", name); + } - public AtlasClassificationDef getClassificationDefByGuid(String guid) { - return classificationDefs.getTypeDefByGuid(guid); - } + if (name != null) { + enumDefs.removeTypeDefByName(name); + structDefs.removeTypeDefByName(name); + classificationDefs.removeTypeDefByName(name); + entityDefs.removeTypeDefByName(name); + } - public AtlasClassificationDef getClassificationDefByName(String name) { - return classificationDefs.getTypeDefByName(name); + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasTypeRegistry.removeEnumDefByName({})", name); + } } - public void removeClassificationDefByGuid(String guid) { + public void addTypes(Collection<? extends AtlasBaseTypeDef> typeDefs) throws AtlasBaseException { if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.removeClassificationDefByGuid({})", guid); + LOG.debug("==> AtlasTypeRegistry.addTypes(length={})", (typeDefs == null ? 0 : typeDefs.size())); } - AtlasClassificationDef classificationDef = classificationDefs.getTypeDefByGuid(guid); - - if (classificationDef != null) { - classificationDefs.removeTypeDefByGuid(guid); + if (CollectionUtils.isNotEmpty(typeDefs)) { + for (AtlasBaseTypeDef typeDef : typeDefs) { + addType(typeDef); + } } if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.removeClassificationDefByGuid({})", guid); + LOG.debug("<== AtlasTypeRegistry.addTypes(length={})", (typeDefs == null ? 0 : typeDefs.size())); } } - public void removeClassificationDefByName(String name) { + public void addTypesWithNoRefResolve(Collection<? extends AtlasBaseTypeDef> typeDefs) { if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.removeClassificationDefByName({})", name); + LOG.debug("==> AtlasTypeRegistry.addTypes(length={})", (typeDefs == null ? 0 : typeDefs.size())); } - AtlasClassificationDef classificationDef = classificationDefs.getTypeDefByName(name); - - if (classificationDef != null) { - classificationDefs.removeTypeDefByName(name); + if (CollectionUtils.isNotEmpty(typeDefs)) { + for (AtlasBaseTypeDef typeDef : typeDefs) { + addTypeWithNoRefResolve(typeDef); + } } if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.removeClassificationDefByName({})", name); + LOG.debug("<== AtlasTypeRegistry.addTypes(length={})", (typeDefs == null ? 0 : typeDefs.size())); } } - - public void addEntityDefWithNoRefResolve(AtlasEntityDef entityDef) { + public void updateTypes(Collection<? extends AtlasBaseTypeDef> typeDefs) throws AtlasBaseException { if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.addEntityDefWithNoRefResolve({})", entityDef); + LOG.debug("==> AtlasTypeRegistry.updateTypes(length={})", (typeDefs == null ? 0 : typeDefs.size())); } - entityDefs.addType(entityDef, new AtlasEntityType(entityDef)); + if (CollectionUtils.isNotEmpty(typeDefs)) { + for (AtlasBaseTypeDef typeDef : typeDefs) { + updateType(typeDef); + } + } if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.addEntityDefWithNoRefResolve({})", entityDef); + LOG.debug("<== AtlasTypeRegistry.updateTypes(length={})", (typeDefs == null ? 0 : typeDefs.size())); } } - public void addEntityDef(AtlasEntityDef entityDef) throws AtlasBaseException { + public void updateTypesWithNoRefResolve(Collection<? extends AtlasBaseTypeDef> typeDefs) { if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.addEntityDef({})", entityDef); + LOG.debug("==> AtlasTypeRegistry.updateTypesWithNoRefResolve(length={})", (typeDefs == null ? 0 : typeDefs.size())); } - entityDefs.addType(entityDef, new AtlasEntityType(entityDef, this)); + if (CollectionUtils.isNotEmpty(typeDefs)) { + for (AtlasBaseTypeDef typeDef : typeDefs) { + updateTypeWithNoRefResolve(typeDef); + } + } if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.addEntityDef({})", entityDef); + LOG.debug("<== AtlasTypeRegistry.updateTypesWithNoRefResolve(length={})", (typeDefs == null ? 0 : typeDefs.size())); } } - public Collection<AtlasEntityDef> getAllEntityDefs() { return entityDefs.getAll(); } - public AtlasEntityDef getEntityDefByGuid(String guid) { - return entityDefs.getTypeDefByGuid(guid); + public Collection<AtlasEnumDef> getAllEnumDefs() { return enumDefs.getAll(); } + + public AtlasEnumDef getEnumDefByGuid(String guid) { + return enumDefs.getTypeDefByGuid(guid); } - public AtlasEntityDef getEntityDefByName(String name) { - return entityDefs.getTypeDefByName(name); + public AtlasEnumDef getEnumDefByName(String name) { + return enumDefs.getTypeDefByName(name); } - public void removeEntityDefByGuid(String guid) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.removeEntityDefByGuid({})", guid); - } - AtlasEntityDef entityDef = entityDefs.getTypeDefByGuid(guid); + public Collection<AtlasStructDef> getAllStructDefs() { return structDefs.getAll(); } - if (entityDef != null) { - entityDefs.removeTypeDefByGuid(guid); - } + public AtlasStructDef getStructDefByGuid(String guid) { + return structDefs.getTypeDefByGuid(guid); + } - if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.removeEntityDefByGuid({})", guid); - } + public AtlasStructDef getStructDefByName(String name) { return structDefs.getTypeDefByName(name); } + + + public Collection<AtlasClassificationDef> getAllClassificationDefs() { return classificationDefs.getAll(); } + + public AtlasClassificationDef getClassificationDefByGuid(String guid) { + return classificationDefs.getTypeDefByGuid(guid); } - public void removeEntityDefByName(String name) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasTypeRegistry.removeEntityDefByName({})", name); - } + public AtlasClassificationDef getClassificationDefByName(String name) { + return classificationDefs.getTypeDefByName(name); + } - AtlasEntityDef entityDef = entityDefs.getTypeDefByName(name); - if (entityDef != null) { - entityDefs.removeTypeDefByName(name); - } + public Collection<AtlasEntityDef> getAllEntityDefs() { return entityDefs.getAll(); } - if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasTypeRegistry.removeEntityDefByName({})", name); + public AtlasEntityDef getEntityDefByGuid(String guid) { + return entityDefs.getTypeDefByGuid(guid); + } + + public AtlasEntityDef getEntityDefByName(String name) { + return entityDefs.getTypeDefByName(name); + } + + + public void resolveReferences() throws AtlasBaseException { + for (Map.Entry<String, AtlasType> e : allTypes.entrySet()) { + e.getValue().resolveReferences(this); } } + private void registerType(AtlasType dataType) { allTypes.put(dataType.getTypeName(), dataType); } @@ -378,6 +503,7 @@ public class AtlasTypeRegistry { allTypes.remove(typeName); } + class TypeDefCache<T extends AtlasBaseTypeDef> { private final AtlasTypeRegistry typeRegistry; private final Map<String, T> typeDefGuidMap = new ConcurrentHashMap<String, T>(); @@ -418,24 +544,25 @@ public class AtlasTypeRegistry { } public void removeTypeDefByGuid(String guid) { - T typeDef = guid != null ? typeDefGuidMap.remove(guid) : null; + T typeDef = guid != null ? typeDefGuidMap.remove(guid) : null; + String name = typeDef != null ? typeDef.getName() : null; - if (typeDef != null) { - if (StringUtils.isNotEmpty(typeDef.getName())) { - typeDefNameMap.remove(typeDef.getName()); - typeRegistry.unregisterTypeByName(typeDef.getName()); - } + if (name != null) { + typeDefNameMap.remove(name); + typeRegistry.unregisterTypeByName(name); } } public void removeTypeDefByName(String name) { - T typeDef = name != null ? typeDefNameMap.get(name) : null; + T typeDef = name != null ? typeDefNameMap.get(name) : null; + String guid = typeDef != null ? typeDef.getGuid() : null; - if (typeDef != null) { - if (StringUtils.isNotEmpty(typeDef.getGuid())) { - typeDefGuidMap.remove(typeDef.getGuid()); - typeRegistry.unregisterTypeByName(typeDef.getName()); - } + if (guid != null) { + typeDefGuidMap.remove(guid); + } + + if (name != null) { + typeRegistry.unregisterTypeByName(name); } } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e5fe91a4/intg/src/test/java/org/apache/atlas/model/ModelTestUtil.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/model/ModelTestUtil.java b/intg/src/test/java/org/apache/atlas/model/ModelTestUtil.java index fcd22f8..e2db68e 100644 --- a/intg/src/test/java/org/apache/atlas/model/ModelTestUtil.java +++ b/intg/src/test/java/org/apache/atlas/model/ModelTestUtil.java @@ -157,7 +157,13 @@ public final class ModelTestUtil { ret.setDefaultValue(ret.getElementDefs().get(idxDefault).getValue()); } - typesRegistry.addEnumDef(ret); + try { + typesRegistry.addType(ret); + } catch (AtlasBaseException excp) { + LOG.error("failed to create enum-def", excp); + + ret = null; + } return ret; } @@ -176,7 +182,7 @@ public final class ModelTestUtil { ret.setAttributeDefs(newAttributeDefsWithAllBuiltInTypes(PREFIX_ATTRIBUTE_NAME)); try { - typesRegistry.addStructDef(ret); + typesRegistry.addType(ret); } catch (AtlasBaseException excp) { LOG.error("failed to create struct-def", excp); @@ -214,7 +220,7 @@ public final class ModelTestUtil { } try { - typesRegistry.addEntityDef(ret); + typesRegistry.addType(ret); } catch (AtlasBaseException excp) { LOG.error("failed to create entity-def", excp); @@ -261,7 +267,7 @@ public final class ModelTestUtil { } try { - typesRegistry.addClassificationDef(ret); + typesRegistry.addType(ret); } catch (AtlasBaseException excp) { LOG.error("failed to create classification-def", excp); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e5fe91a4/intg/src/test/java/org/apache/atlas/type/TestAtlasEntityType.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/type/TestAtlasEntityType.java b/intg/src/test/java/org/apache/atlas/type/TestAtlasEntityType.java index 8139fd4..5b38e3c 100644 --- a/intg/src/test/java/org/apache/atlas/type/TestAtlasEntityType.java +++ b/intg/src/test/java/org/apache/atlas/type/TestAtlasEntityType.java @@ -35,7 +35,7 @@ public class TestAtlasEntityType { private final List<Object> invalidValues = new ArrayList<Object>(); { - entityType = getEntityType(ModelTestUtil.getEntityDefWithSuperTypes()); + entityType = getEntityType(ModelTestUtil.getEntityDefWithSuperTypes()); AtlasEntity invalidValue1 = entityType.createDefaultValue(); AtlasEntity invalidValue2 = entityType.createDefaultValue(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e5fe91a4/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 1fb73e4..47b0d4d 100644 --- a/release-log.txt +++ b/release-log.txt @@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai) ALL CHANGES: +ATLAS-1225 Optimize AtlasTypeDefGraphStore to use typeRegistry (mneethiraj via sumasai) ATLAS-1221 build failure in windows SyntaxError: invalid syntax (zhangqiang2 via shwethags) ATLAS-1226 Servlet init-params in web.xml are unused (mneethiraj via shwethags) ATLAS-1224 Minor fixes for hive and falcon bridge twiki (ayubkhan via sumasai) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e5fe91a4/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java index 9c0e569..14cc7c4 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java @@ -32,12 +32,15 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasStructDefs; import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.repository.util.FilterUtil; import org.apache.atlas.store.AtlasTypeDefStore; +import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.util.TypeDefSorter; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.Predicate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; @@ -49,14 +52,39 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore { private static final Logger LOG = LoggerFactory.getLogger(AtlasTypeDefGraphStore.class); - protected AtlasEnumDefStore enumDefStore; - protected AtlasStructDefStore structDefStore; - protected AtlasClassificationDefStore classificationDefStore; - protected AtlasEntityDefStore entityDefStore; + private AtlasTypeRegistry typeRegistry; + private AtlasEnumDefStore enumDefStore; + private AtlasStructDefStore structDefStore; + private AtlasClassificationDefStore classificationDefStore; + private AtlasEntityDefStore entityDefStore; protected AtlasTypeDefGraphStore() { } + protected void init(AtlasEnumDefStore enumDefStore, + AtlasStructDefStore structDefStore, + AtlasClassificationDefStore classificationDefStore, + AtlasEntityDefStore entityDefStore) throws AtlasBaseException { + AtlasTypeRegistry typeRegistry = new AtlasTypeRegistry(); + + typeRegistry.addTypesWithNoRefResolve(enumDefStore.getAll()); + typeRegistry.addTypesWithNoRefResolve(structDefStore.getAll()); + typeRegistry.addTypesWithNoRefResolve(classificationDefStore.getAll()); + typeRegistry.addTypesWithNoRefResolve(entityDefStore.getAll()); + + typeRegistry.resolveReferences(); + + this.enumDefStore = enumDefStore; + this.structDefStore = structDefStore; + this.classificationDefStore = classificationDefStore; + this.entityDefStore = entityDefStore; + this.typeRegistry = typeRegistry; + } + + public AtlasTypeRegistry getTypeRegistry() { + return typeRegistry; + } + @Override public void init() throws AtlasBaseException { @@ -65,55 +93,87 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore { @Override @GraphTransaction public AtlasEnumDef createEnumDef(AtlasEnumDef enumDef) throws AtlasBaseException { - return enumDefStore.create(enumDef); + AtlasEnumDef ret = enumDefStore.create(enumDef); + + typeRegistry.addType(ret); + + return ret; } @Override @GraphTransaction public List<AtlasEnumDef> createEnumDefs(List<AtlasEnumDef> atlasEnumDefs) throws AtlasBaseException { - return enumDefStore.create(atlasEnumDefs); + List<AtlasEnumDef> ret = enumDefStore.create(atlasEnumDefs); + + typeRegistry.addTypes(ret); + + return ret; } @Override @GraphTransaction public List<AtlasEnumDef> getAllEnumDefs() throws AtlasBaseException { - return enumDefStore.getAll(); + List<AtlasEnumDef> ret = null; + + Collection<AtlasEnumDef> enumDefs = typeRegistry.getAllEnumDefs(); + + if (enumDefs != null) { + ret = new ArrayList<>(enumDefs); + } + + return ret; } @Override @GraphTransaction public AtlasEnumDef getEnumDefByName(String name) throws AtlasBaseException { - return enumDefStore.getByName(name); + AtlasEnumDef ret = typeRegistry.getEnumDefByName(name); + + return ret; } @Override @GraphTransaction public AtlasEnumDef getEnumDefByGuid(String guid) throws AtlasBaseException { - return enumDefStore.getByGuid(guid); + AtlasEnumDef ret = typeRegistry.getEnumDefByGuid(guid); + + return ret; } @Override @GraphTransaction public AtlasEnumDef updateEnumDefByName(String name, AtlasEnumDef enumDef) throws AtlasBaseException { - return enumDefStore.updateByName(name, enumDef); + AtlasEnumDef ret = enumDefStore.updateByName(name, enumDef); + + typeRegistry.updateTypeByName(name, ret); + + return ret; } @Override @GraphTransaction public AtlasEnumDef updateEnumDefByGuid(String guid, AtlasEnumDef enumDef) throws AtlasBaseException { - return enumDefStore.updateByGuid(guid, enumDef); + AtlasEnumDef ret = enumDefStore.updateByGuid(guid, enumDef); + + typeRegistry.updateTypeByGuid(guid, ret); + + return ret; } @Override @GraphTransaction public void deleteEnumDefByName(String name) throws AtlasBaseException { enumDefStore.deleteByName(name); + + typeRegistry.removeTypeByName(name); } @Override @GraphTransaction public void deleteEnumDefByGuid(String guid) throws AtlasBaseException { enumDefStore.deleteByGuid(guid); + + typeRegistry.removeTypeByGuid(guid); } @Override @@ -125,55 +185,87 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore { @Override @GraphTransaction public AtlasStructDef createStructDef(AtlasStructDef structDef) throws AtlasBaseException { - return structDefStore.create(structDef); + AtlasStructDef ret = structDefStore.create(structDef); + + typeRegistry.addType(structDef); + + return ret; } @Override @GraphTransaction public List<AtlasStructDef> createStructDefs(List<AtlasStructDef> structDefs) throws AtlasBaseException { - return structDefStore.create(structDefs); + List<AtlasStructDef> ret = structDefStore.create(structDefs); + + typeRegistry.addTypes(ret); + + return ret; } @Override @GraphTransaction public List<AtlasStructDef> getAllStructDefs() throws AtlasBaseException { - return structDefStore.getAll(); + List<AtlasStructDef> ret = null; + + Collection<AtlasStructDef> structDefs = typeRegistry.getAllStructDefs(); + + if (structDefs != null) { + ret = new ArrayList<>(structDefs); + } + + return ret; } @Override @GraphTransaction public AtlasStructDef getStructDefByName(String name) throws AtlasBaseException { - return structDefStore.getByName(name); + AtlasStructDef ret = typeRegistry.getStructDefByName(name); + + return ret; } @Override @GraphTransaction public AtlasStructDef getStructDefByGuid(String guid) throws AtlasBaseException { - return structDefStore.getByGuid(guid); + AtlasStructDef ret = typeRegistry.getStructDefByGuid(guid); + + return ret; } @Override @GraphTransaction public AtlasStructDef updateStructDefByName(String name, AtlasStructDef structDef) throws AtlasBaseException { - return structDefStore.updateByName(name, structDef); + AtlasStructDef ret = structDefStore.updateByName(name, structDef); + + typeRegistry.updateTypeByName(name, ret); + + return ret; } @Override @GraphTransaction public AtlasStructDef updateStructDefByGuid(String guid, AtlasStructDef structDef) throws AtlasBaseException { - return structDefStore.updateByGuid(guid, structDef); + AtlasStructDef ret = structDefStore.updateByGuid(guid, structDef); + + typeRegistry.updateTypeByGuid(guid, ret); + + return ret; } @Override @GraphTransaction public void deleteStructDefByName(String name) throws AtlasBaseException { structDefStore.deleteByName(name); + + typeRegistry.removeTypeByName(name); } @Override @GraphTransaction public void deleteStructDefByGuid(String guid) throws AtlasBaseException { structDefStore.deleteByGuid(guid); + + typeRegistry.removeTypeByGuid(guid); } @Override @@ -184,56 +276,92 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore { @Override @GraphTransaction - public AtlasClassificationDef createClassificationDef(AtlasClassificationDef classificationDef) throws AtlasBaseException { - return classificationDefStore.create(classificationDef); + public AtlasClassificationDef createClassificationDef(AtlasClassificationDef classificationDef) + throws AtlasBaseException { + AtlasClassificationDef ret = classificationDefStore.create(classificationDef); + + typeRegistry.addType(classificationDef); + + return ret; } @Override @GraphTransaction - public List<AtlasClassificationDef> createClassificationDefs(List<AtlasClassificationDef> classificationDefs) throws AtlasBaseException { - return classificationDefStore.create(classificationDefs); + public List<AtlasClassificationDef> createClassificationDefs(List<AtlasClassificationDef> classificationDefs) + throws AtlasBaseException { + List<AtlasClassificationDef> ret = classificationDefStore.create(classificationDefs); + + typeRegistry.addTypes(ret); + + return ret; } @Override @GraphTransaction public List<AtlasClassificationDef> getAllClassificationDefs() throws AtlasBaseException { - return classificationDefStore.getAll(); + List<AtlasClassificationDef> ret = null; + + Collection<AtlasClassificationDef> classificationDefs = typeRegistry.getAllClassificationDefs(); + + if (classificationDefs != null) { + ret = new ArrayList<>(classificationDefs); + } + + return ret; } @Override @GraphTransaction public AtlasClassificationDef getClassificationDefByName(String name) throws AtlasBaseException { - return classificationDefStore.getByName(name); + AtlasClassificationDef ret = typeRegistry.getClassificationDefByName(name); + + return ret; } @Override @GraphTransaction public AtlasClassificationDef getClassificationDefByGuid(String guid) throws AtlasBaseException { - return classificationDefStore.getByGuid(guid); + AtlasClassificationDef ret = typeRegistry.getClassificationDefByGuid(guid); + + return ret; } @Override @GraphTransaction - public AtlasClassificationDef updateClassificationDefByName(String name, AtlasClassificationDef classificationDef) throws AtlasBaseException { - return classificationDefStore.updateByName(name, classificationDef); + public AtlasClassificationDef updateClassificationDefByName(String name, AtlasClassificationDef classificationDef) + throws AtlasBaseException { + AtlasClassificationDef ret = classificationDefStore.updateByName(name, classificationDef); + + typeRegistry.updateTypeByName(name, ret); + + return ret; } @Override @GraphTransaction - public AtlasClassificationDef updateClassificationDefByGuid(String guid, AtlasClassificationDef classificationDef) throws AtlasBaseException { - return classificationDefStore.updateByGuid(guid, classificationDef); + public AtlasClassificationDef updateClassificationDefByGuid(String guid, AtlasClassificationDef classificationDef) + throws AtlasBaseException { + AtlasClassificationDef ret = classificationDefStore.updateByGuid(guid, classificationDef); + + typeRegistry.updateTypeByGuid(guid, ret); + + return ret; } @Override @GraphTransaction public void deleteClassificationDefByName(String name) throws AtlasBaseException { classificationDefStore.deleteByName(name); + + typeRegistry.removeTypeByName(name); } @Override @GraphTransaction public void deleteClassificationDefByGuid(String guid) throws AtlasBaseException { classificationDefStore.deleteByGuid(guid); + + typeRegistry.removeTypeByGuid(guid); } @Override @@ -244,56 +372,88 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore { @Override @GraphTransaction - public AtlasEntityDef createEntityDefs(AtlasEntityDef entityDef) throws AtlasBaseException { - return entityDefStore.create(entityDef); + public AtlasEntityDef createEntityDef(AtlasEntityDef entityDef) throws AtlasBaseException { + AtlasEntityDef ret = entityDefStore.create(entityDef); + + typeRegistry.addType(ret); + + return ret; } @Override @GraphTransaction public List<AtlasEntityDef> createEntityDefs(List<AtlasEntityDef> entityDefs) throws AtlasBaseException { - return entityDefStore.create(entityDefs); + List<AtlasEntityDef> ret = entityDefStore.create(entityDefs); + + typeRegistry.addTypes(ret); + + return ret; } @Override @GraphTransaction public List<AtlasEntityDef> getAllEntityDefs() throws AtlasBaseException { - return entityDefStore.getAll(); + List<AtlasEntityDef> ret = null; + + Collection<AtlasEntityDef> entityDefs = typeRegistry.getAllEntityDefs(); + + if (entityDefs != null) { + ret = new ArrayList<>(entityDefs); + } + + return ret; } @Override @GraphTransaction public AtlasEntityDef getEntityDefByName(String name) throws AtlasBaseException { - return entityDefStore.getByName(name); + AtlasEntityDef ret = typeRegistry.getEntityDefByName(name); + + return ret; } @Override @GraphTransaction public AtlasEntityDef getEntityDefByGuid(String guid) throws AtlasBaseException { - return entityDefStore.getByGuid(guid); + AtlasEntityDef ret = typeRegistry.getEntityDefByGuid(guid); + + return ret; } @Override @GraphTransaction public AtlasEntityDef updateEntityDefByName(String name, AtlasEntityDef entityDef) throws AtlasBaseException { - return entityDefStore.updateByName(name, entityDef); + AtlasEntityDef ret = entityDefStore.updateByName(name, entityDef); + + typeRegistry.updateTypeByName(name, ret); + + return ret; } @Override @GraphTransaction public AtlasEntityDef updateEntityDefByGuid(String guid, AtlasEntityDef entityDef) throws AtlasBaseException { - return entityDefStore.updateByGuid(guid, entityDef); + AtlasEntityDef ret = entityDefStore.updateByGuid(guid, entityDef); + + typeRegistry.updateTypeByGuid(guid, ret); + + return ret; } @Override @GraphTransaction public void deleteEntityDefByName(String name) throws AtlasBaseException { entityDefStore.deleteByName(name); + + typeRegistry.removeTypeByName(name); } @Override @GraphTransaction public void deleteEntityDefByGuid(String guid) throws AtlasBaseException { entityDefStore.deleteByGuid(guid); + + typeRegistry.removeTypeByGuid(guid); } @Override @@ -302,110 +462,48 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore { return entityDefStore.search(filter); } - private List<? extends AtlasBaseTypeDef> createOrUpdateTypeDefs(List<? extends AtlasBaseTypeDef> typeDefs, boolean isUpdate) { - List<AtlasBaseTypeDef> ret = Collections.emptyList(); - - if (CollectionUtils.isNotEmpty(typeDefs)) { - AtlasBaseTypeDef typeDef = typeDefs.get(0); - if (LOG.isDebugEnabled()) { - if (isUpdate) { - LOG.debug("Updating {} {}", typeDefs.size(), typeDef.getClass().getSimpleName()); - } else { - LOG.debug("Creating {} {}", typeDefs.size(), typeDef.getClass().getSimpleName()); - } - } - - if (typeDef instanceof AtlasEntityDef) { - List<AtlasEntityDef> entityDefs = TypeDefSorter.sortTypes((List<AtlasEntityDef>) typeDefs); - try { - if (isUpdate) { - return entityDefStore.update((List<AtlasEntityDef>) typeDefs); - } else { - return entityDefStore.create((List<AtlasEntityDef>) typeDefs); - } - } catch (AtlasBaseException ex) { - LOG.error("Failed to " + (isUpdate ? "update" : "create") + " EntityDefs", ex); - } - } else if (typeDef instanceof AtlasClassificationDef) { - List<AtlasClassificationDef> classificationDefs = - TypeDefSorter.sortTypes((List<AtlasClassificationDef>) typeDefs); - try { - if (isUpdate) { - return classificationDefStore.update((List<AtlasClassificationDef>) typeDefs); - } else { - return classificationDefStore.create((List<AtlasClassificationDef>) typeDefs); - } - } catch (AtlasBaseException ex) { - LOG.error("Failed to " + (isUpdate ? "update" : "create") + " ClassificationDefs", ex); - } - - } else if (typeDef instanceof AtlasStructDef) { - try { - if (isUpdate) { - return structDefStore.update((List<AtlasStructDef>) typeDefs); - } else { - return structDefStore.create((List<AtlasStructDef>) typeDefs); - } - } catch (AtlasBaseException ex) { - LOG.error("Failed to " + (isUpdate ? "update" : "create") + " StructDefs", ex); - } - } else if (typeDef instanceof AtlasEnumDef) { - try { - if (isUpdate) { - return enumDefStore.update((List<AtlasEnumDef>) typeDefs); - } else { - return enumDefStore.create((List<AtlasEnumDef>) typeDefs); - } - } catch (AtlasBaseException ex) { - LOG.error("Failed to " + (isUpdate ? "update" : "create") + " EnumDefs", ex); - } - } - } - return ret; - } - @Override @GraphTransaction public AtlasTypesDef createTypesDef(AtlasTypesDef typesDef) throws AtlasBaseException { - AtlasTypesDef createdTypeDefs = new AtlasTypesDef(); - LOG.info("Creating EnumDefs"); - List<? extends AtlasBaseTypeDef> createdEnumDefs = createOrUpdateTypeDefs(typesDef.getEnumDefs(), false); - LOG.info("Creating StructDefs"); - List<? extends AtlasBaseTypeDef> createdStructDefs = createOrUpdateTypeDefs(typesDef.getStructDefs(), false); - LOG.info("Creating ClassificationDefs"); - List<? extends AtlasBaseTypeDef> createdClassificationDefs = createOrUpdateTypeDefs(typesDef.getClassificationDefs(), false); - LOG.info("Creating EntityDefs"); - List<? extends AtlasBaseTypeDef> createdEntityDefs = createOrUpdateTypeDefs(typesDef.getEntityDefs(), false); - - typesDef.setEnumDefs((List<AtlasEnumDef>) createdEnumDefs); - typesDef.setStructDefs((List<AtlasStructDef>) createdStructDefs); - typesDef.setClassificationDefs((List<AtlasClassificationDef>) createdClassificationDefs); - typesDef.setEntityDefs((List<AtlasEntityDef>) createdEntityDefs); + List<AtlasEnumDef> enumDefs = enumDefStore.create(typesDef.getEnumDefs()); + List<AtlasStructDef> structDefs = structDefStore.create(typesDef.getStructDefs()); + List<AtlasClassificationDef> classifiDefs = classificationDefStore.create(typesDef.getClassificationDefs()); + List<AtlasEntityDef> entityDefs = entityDefStore.create(typesDef.getEntityDefs()); - return typesDef; + // typeRegistry should be updated only after resovleReferences() returns success; until then use a temp registry + typeRegistry.addTypes(enumDefs); + typeRegistry.addTypes(structDefs); + typeRegistry.addTypes(classifiDefs); + typeRegistry.addTypes(entityDefs); + + typeRegistry.resolveReferences(); + + AtlasTypesDef ret = new AtlasTypesDef(enumDefs, structDefs, classifiDefs, entityDefs); + + return ret; } @Override @GraphTransaction public AtlasTypesDef updateTypesDef(AtlasTypesDef typesDef) throws AtlasBaseException { - AtlasTypesDef createdTypeDefs = new AtlasTypesDef(); - LOG.info("Updating EnumDefs"); - List<? extends AtlasBaseTypeDef> updatedEnumDefs = createOrUpdateTypeDefs(typesDef.getEnumDefs(), true); - LOG.info("Updating StructDefs"); - List<? extends AtlasBaseTypeDef> updatedStructDefs = createOrUpdateTypeDefs(typesDef.getStructDefs(), true); - LOG.info("Updating ClassificationDefs"); - List<? extends AtlasBaseTypeDef> updatedClassficationDefs = createOrUpdateTypeDefs(typesDef.getClassificationDefs(), true); - LOG.info("Updating EntityDefs"); - List<? extends AtlasBaseTypeDef> updatedEntityDefs = createOrUpdateTypeDefs(typesDef.getEntityDefs(), true); - - typesDef.setEnumDefs((List<AtlasEnumDef>) updatedEnumDefs); - typesDef.setStructDefs((List<AtlasStructDef>) updatedStructDefs); - typesDef.setClassificationDefs((List<AtlasClassificationDef>) updatedClassficationDefs); - typesDef.setEntityDefs((List<AtlasEntityDef>) updatedEntityDefs); + List<AtlasEnumDef> enumDefs = enumDefStore.update(typesDef.getEnumDefs()); + List<AtlasStructDef> structDefs = structDefStore.update(typesDef.getStructDefs()); + List<AtlasClassificationDef> classifiDefs = classificationDefStore.update(typesDef.getClassificationDefs()); + List<AtlasEntityDef> entityDefs = entityDefStore.update(typesDef.getEntityDefs()); - return typesDef; + // typeRegistry should be updated only after resovleReferences() returns success; until then use a temp registry + typeRegistry.updateTypes(enumDefs); + typeRegistry.updateTypes(structDefs); + typeRegistry.updateTypes(classifiDefs); + typeRegistry.updateTypes(entityDefs); + + typeRegistry.resolveReferences(); + + AtlasTypesDef ret = new AtlasTypesDef(enumDefs, structDefs, classifiDefs, entityDefs); + + return ret; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e5fe91a4/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasClassificationDefStoreV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasClassificationDefStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasClassificationDefStoreV1.java index 9afccbe..78cbd63 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasClassificationDefStoreV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasClassificationDefStoreV1.java @@ -31,8 +31,6 @@ import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import sun.security.provider.certpath.Vertex; - import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedList; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e5fe91a4/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java index c4e7dcd..ea1b03b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java @@ -21,6 +21,7 @@ import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.SearchFilter; import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; + import org.apache.atlas.model.typedef.AtlasStructDef.AtlasStructDefs; import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graphdb.AtlasVertex; @@ -381,7 +382,7 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore { String propertyKey = AtlasGraphUtilsV1.getPropertyKey(ret, attrName); String attribJson = atlasVertex.getProperty(propertyKey, String.class); - attributeDefs.add(toAttributeDefFromJson(attribJson)); + attributeDefs.add(toAttributeDefFromJson(AtlasType.fromJson(attribJson, Map.class))); } } ret.setAttributeDefs(attributeDefs); @@ -412,14 +413,17 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore { } private static String toJsonFromAttributeDef(AtlasAttributeDef attributeDef) { + Boolean isComposite = Boolean.FALSE; + String reverseAttribName = null; + Map<String, Object> attribInfo = new HashMap<String, Object>(); attribInfo.put("name", attributeDef.getName()); attribInfo.put("dataType", attributeDef.getTypeName()); attribInfo.put("isUnique", attributeDef.isUnique()); attribInfo.put("isIndexable", attributeDef.isIndexable()); - attribInfo.put("isComposite", Boolean.FALSE); - attribInfo.put("reverseAttributeName", ""); + attribInfo.put("isComposite", isComposite); + attribInfo.put("reverseAttributeName", reverseAttribName); Map<String, Object> multiplicity = new HashMap<String, Object>(); multiplicity.put("lower", attributeDef.getValuesMinCount()); multiplicity.put("upper", attributeDef.getValuesMaxCount()); @@ -430,23 +434,23 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore { return AtlasType.toJson(attribInfo); } - private static AtlasAttributeDef toAttributeDefFromJson(String json) { - Map attribInfo = AtlasType.fromJson(json, Map.class); - + private static AtlasAttributeDef toAttributeDefFromJson(Map attribInfo) { AtlasAttributeDef ret = new AtlasAttributeDef(); ret.setName((String) attribInfo.get("name")); ret.setTypeName((String) attribInfo.get("dataType")); ret.setUnique((Boolean) attribInfo.get("isUnique")); ret.setIndexable((Boolean) attribInfo.get("isIndexable")); + /* - attributeMap.put("isComposite", isComposite); - attributeMap.put("reverseAttributeName", reverseAttributeName); + String reverseAttribName = (String)attribInfo.get("reverseAttributeName"); + Boolean isComposite = (Boolean) attribInfo.get("isComposite"); */ - Map multiplicity = AtlasType.fromJson((String) attribInfo.get("multiplicity"), Map.class); - Number minCount = (Number) multiplicity.get("lower"); - Number maxCount = (Number) multiplicity.get("upper"); - Boolean isUnique = (Boolean) multiplicity.get("isUnique"); + Map multiplicity = AtlasType.fromJson((String) attribInfo.get("multiplicity"), Map.class); + Number minCount = (Number) multiplicity.get("lower"); + Number maxCount = (Number) multiplicity.get("upper"); + Boolean isUnique = (Boolean) multiplicity.get("isUnique"); + if (minCount == null || minCount.intValue() == 0) { ret.setOptional(true); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e5fe91a4/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java index 9726530..4b50aac 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java @@ -60,15 +60,29 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore { public AtlasTypeDefGraphStoreV1() { super(); - enumDefStore = new AtlasEnumDefStoreV1(this); - structDefStore = new AtlasStructDefStoreV1(this); - classificationDefStore = new AtlasClassificationDefStoreV1(this); - entityDefStore = new AtlasEntityDefStoreV1(this); + LOG.info("==> AtlasTypeDefGraphStoreV1()"); + + try { + init(); + } catch(AtlasBaseException excp) { + LOG.error("failed to initialize types from graph store", excp); + } + + LOG.info("<== AtlasTypeDefGraphStoreV1()"); } @Override - public void init() { + public void init() throws AtlasBaseException { + LOG.info("==> AtlasTypeDefGraphStoreV1.init()"); + + super.init(); + + super.init(new AtlasEnumDefStoreV1(this), + new AtlasStructDefStoreV1(this), + new AtlasClassificationDefStoreV1(this), + new AtlasEntityDefStoreV1(this)); + LOG.info("<== AtlasTypeDefGraphStoreV1.init()"); } public AtlasGraph getAtlasGraph() { return atlasGraph; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e5fe91a4/webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java index 728bec3..889c3dd 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java @@ -350,7 +350,7 @@ public class TypesREST { AtlasEntityDef ret = null; try { - ret = typeDefStore.createEntityDefs(entityDef); + ret = typeDefStore.createEntityDef(entityDef); return ret; } catch (AtlasBaseException ex) { throw new WebApplicationException(Servlets.getErrorResponse(ex, Response.Status.BAD_REQUEST));
