Repository: incubator-atlas Updated Branches: refs/heads/master d0cb5c37d -> 2720af614
ATLAS-1673: Type deletion should check for reference & instances Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/2720af61 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/2720af61 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/2720af61 Branch: refs/heads/master Commit: 2720af61499db1f4bd42d1dfa2cec766aeabd702 Parents: d0cb5c3 Author: apoorvnaik <[email protected]> Authored: Mon Mar 20 10:56:19 2017 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Mon Apr 3 07:28:05 2017 -0700 ---------------------------------------------------------------------- .../apache/atlas/type/AtlasTypeRegistry.java | 59 +++++++++++++++++++- .../graph/GraphBackedSearchIndexer.java | 18 +----- .../store/graph/AtlasTypeDefGraphStore.java | 3 + .../graph/v1/AtlasClassificationDefStoreV1.java | 10 ++++ .../store/graph/v1/AtlasEntityDefStoreV1.java | 10 ++++ .../store/graph/v1/AtlasGraphUtilsV1.java | 16 ++++++ .../store/graph/v1/AtlasStructDefStoreV1.java | 10 ++++ 7 files changed, 110 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2720af61/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java index 2c80cf5..ea2a703 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java @@ -512,6 +512,63 @@ public class AtlasTypeRegistry { } } + public void removeTypesDef(AtlasTypesDef typesDef) throws AtlasBaseException { + if (null != typesDef && !typesDef.isEmpty()) { + removeTypesWithNoRefResolve(typesDef.getEnumDefs()); + removeTypesWithNoRefResolve(typesDef.getStructDefs()); + removeTypesWithNoRefResolve(typesDef.getClassificationDefs()); + removeTypesWithNoRefResolve(typesDef.getEntityDefs()); + + resolveReferences(); + } + } + + private void removeTypesWithNoRefResolve(Collection<? extends AtlasBaseTypeDef> typeDefs) { + if (CollectionUtils.isNotEmpty(typeDefs)) { + for (AtlasBaseTypeDef typeDef : typeDefs) { + if (StringUtils.isNotEmpty(typeDef.getGuid())) { + removeTypeByGuidWithNoRefResolve(typeDef); + } else { + removeTypeByNameWithNoRefResolve(typeDef); + } + } + } + } + + private void removeTypeByNameWithNoRefResolve(AtlasBaseTypeDef typeDef) { + switch (typeDef.getCategory()) { + case ENUM: + registryData.enumDefs.removeTypeDefByName(typeDef.getName()); + break; + case STRUCT: + registryData.structDefs.removeTypeDefByName(typeDef.getName()); + break; + case CLASSIFICATION: + registryData.classificationDefs.removeTypeDefByName(typeDef.getName()); + break; + case ENTITY: + registryData.entityDefs.removeTypeDefByName(typeDef.getName()); + break; + } + } + + private void removeTypeByGuidWithNoRefResolve(AtlasBaseTypeDef typeDef) { + switch (typeDef.getCategory()) { + case ENUM: + registryData.enumDefs.removeTypeDefByGuid(typeDef.getGuid()); + break; + case STRUCT: + registryData.structDefs.removeTypeDefByGuid(typeDef.getGuid()); + break; + case CLASSIFICATION: + registryData.classificationDefs.removeTypeDefByGuid(typeDef.getGuid()); + break; + case ENTITY: + registryData.entityDefs.removeTypeDefByGuid(typeDef.getGuid()); + break; + } + } + public void removeTypeByGuid(String guid) throws AtlasBaseException { if (LOG.isDebugEnabled()) { LOG.debug("==> AtlasTypeRegistry.removeTypeByGuid({})", guid); @@ -892,7 +949,7 @@ class TypeCache { public void removeTypeByName(String name) { if (name != null) { - typeNameMap.get(name); + typeNameMap.remove(name); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2720af61/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java index 43d6389..47dccf1 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java @@ -20,7 +20,6 @@ package org.apache.atlas.repository.graph; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; - import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasException; @@ -62,6 +61,7 @@ import org.apache.commons.configuration.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.inject.Inject; import java.math.BigDecimal; import java.math.BigInteger; import java.util.ArrayList; @@ -70,25 +70,13 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import javax.inject.Inject; - -import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_BIGDECIMAL; -import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_BIGINTEGER; -import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_BOOLEAN; -import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_BYTE; -import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_DATE; -import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_DOUBLE; -import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_FLOAT; -import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_INT; -import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_LONG; -import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_SHORT; -import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_STRING; +import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.*; /** * Adds index for properties of a given type when its added before any instances are added. */ -public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChangeHandler, +public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChangeHandler, TypeDefChangeListener { private static final Logger LOG = LoggerFactory.getLogger(GraphBackedSearchIndexer.class); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2720af61/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java index b3e2f75..82465bf 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java @@ -493,6 +493,9 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore, Activ } } + // Remove all from + ttr.removeTypesDef(typesDef); + if (LOG.isDebugEnabled()) { LOG.debug("<== AtlasTypeDefGraphStore.deleteTypesDef(enums={}, structs={}, classfications={}, entities={})", CollectionUtils.size(typesDef.getEnumDefs()), http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2720af61/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasClassificationDefStoreV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasClassificationDefStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasClassificationDefStoreV1.java index 7ef3558..d94aa21 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasClassificationDefStoreV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasClassificationDefStoreV1.java @@ -258,6 +258,10 @@ public class AtlasClassificationDefStoreV1 extends AtlasAbstractDefStoreV1 imple AtlasVertex ret = typeDefStore.findTypeVertexByNameAndCategory(name, TypeCategory.TRAIT); + if (AtlasGraphUtilsV1.typeHasInstanceVertex(name)) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_HAS_REFERENCES, name); + } + if (ret == null) { throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, name); } @@ -300,6 +304,12 @@ public class AtlasClassificationDefStoreV1 extends AtlasAbstractDefStoreV1 imple AtlasVertex ret = typeDefStore.findTypeVertexByGuidAndCategory(guid, TypeCategory.TRAIT); + String typeName = AtlasGraphUtilsV1.getProperty(ret, Constants.TYPENAME_PROPERTY_KEY, String.class); + + if (AtlasGraphUtilsV1.typeHasInstanceVertex(typeName)) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_HAS_REFERENCES, typeName); + } + if (ret == null) { throw new AtlasBaseException(AtlasErrorCode.TYPE_GUID_NOT_FOUND, guid); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2720af61/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityDefStoreV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityDefStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityDefStoreV1.java index 0771a3a..096f99b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityDefStoreV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityDefStoreV1.java @@ -256,6 +256,10 @@ public class AtlasEntityDefStoreV1 extends AtlasAbstractDefStoreV1 implements At AtlasVertex ret = typeDefStore.findTypeVertexByNameAndCategory(name, TypeCategory.CLASS); + if (AtlasGraphUtilsV1.typeHasInstanceVertex(name)) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_HAS_REFERENCES, name); + } + if (ret == null) { throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, name); } @@ -298,6 +302,12 @@ public class AtlasEntityDefStoreV1 extends AtlasAbstractDefStoreV1 implements At AtlasVertex ret = typeDefStore.findTypeVertexByGuidAndCategory(guid, TypeCategory.CLASS); + String typeName = AtlasGraphUtilsV1.getProperty(ret, Constants.TYPENAME_PROPERTY_KEY, String.class); + + if (AtlasGraphUtilsV1.typeHasInstanceVertex(typeName)) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_HAS_REFERENCES, typeName); + } + if (ret == null) { throw new AtlasBaseException(AtlasErrorCode.TYPE_GUID_NOT_FOUND, guid); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2720af61/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java index 13b4e9b..560b338 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java @@ -234,6 +234,22 @@ public class AtlasGraphUtilsV1 { return vertex; } + public static boolean typeHasInstanceVertex(String typeName) throws AtlasBaseException { + AtlasGraphQuery query = AtlasGraphProvider.getGraphInstance() + .query() + .has(Constants.TYPE_NAME_PROPERTY_KEY, AtlasGraphQuery.ComparisionOperator.EQUAL, typeName); + + Iterator<AtlasVertex> results = query.vertices().iterator(); + + boolean hasInstanceVertex = results != null && results.hasNext(); + + if (LOG.isDebugEnabled()) { + LOG.debug("typeName {} has instance vertex {}", typeName, hasInstanceVertex); + } + + return hasInstanceVertex; + } + public static AtlasVertex findByTypeAndPropertyName(String typeName, String propertyName, Object attrVal) { AtlasGraphQuery query = AtlasGraphProvider.getGraphInstance().query() .has(Constants.ENTITY_TYPE_PROPERTY_KEY, typeName) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2720af61/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java index 2bc36da..f154555 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java @@ -267,6 +267,10 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At AtlasVertex ret = typeDefStore.findTypeVertexByNameAndCategory(name, TypeCategory.STRUCT); + if (AtlasGraphUtilsV1.typeHasInstanceVertex(name)) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_HAS_REFERENCES, name); + } + if (ret == null) { throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, name); } @@ -309,6 +313,12 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At AtlasVertex ret = typeDefStore.findTypeVertexByGuidAndCategory(guid, TypeCategory.STRUCT); + String typeName = AtlasGraphUtilsV1.getProperty(ret, Constants.TYPENAME_PROPERTY_KEY, String.class); + + if (AtlasGraphUtilsV1.typeHasInstanceVertex(typeName)) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_HAS_REFERENCES, typeName); + } + if (ret == null) { throw new AtlasBaseException(AtlasErrorCode.TYPE_GUID_NOT_FOUND, guid); }
