Repository: incubator-atlas Updated Branches: refs/heads/master a96424a1a -> 5a4dd2e72
ATLAS-1235: TypesREST.update() does not update fields like description Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/5a4dd2e7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/5a4dd2e7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/5a4dd2e7 Branch: refs/heads/master Commit: 5a4dd2e724bdca8db02dd4b9b6d59eb19296de95 Parents: a96424a Author: Madhan Neethiraj <[email protected]> Authored: Thu Oct 20 16:13:13 2016 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Sat Oct 22 11:17:13 2016 -0700 ---------------------------------------------------------------------- .../atlas/type/AtlasClassificationType.java | 52 +++- .../org/apache/atlas/type/AtlasEntityType.java | 50 +++- .../atlas/type/TestAtlasTypeRegistry.java | 270 +++++++++++++++++++ .../graph/v1/AtlasClassificationDefStoreV1.java | 10 +- .../store/graph/v1/AtlasEntityDefStoreV1.java | 11 +- .../store/graph/v1/AtlasStructDefStoreV1.java | 58 +++- .../graph/v1/AtlasTypeDefGraphStoreV1.java | 76 +++++- 7 files changed, 482 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5a4dd2e7/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java index 6299265..56ff803 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java @@ -65,22 +65,13 @@ public class AtlasClassificationType extends AtlasStructType { super.resolveReferences(typeRegistry); List<AtlasClassificationType> s = new ArrayList<AtlasClassificationType>(); - Set<String> allS = new HashSet<String>(); + Set<String> allS = getAllSuperTypes(typeRegistry); for (String superTypeName : classificationDef.getSuperTypes()) { AtlasType superType = typeRegistry.getType(superTypeName); if (superType instanceof AtlasClassificationType) { - AtlasClassificationType superClassificationType = (AtlasClassificationType)superType; - - superClassificationType.resolveReferences(typeRegistry); - - s.add(superClassificationType); - allS.add(superTypeName); - - if (CollectionUtils.isNotEmpty(superClassificationType.getAllSuperTypes())) { - allS.addAll(superClassificationType.getAllSuperTypes()); - } + s.add((AtlasClassificationType)superType); } else { throw new AtlasBaseException(superTypeName + ": incompatible supertype in classification " + classificationDef.getName()); @@ -88,7 +79,7 @@ public class AtlasClassificationType extends AtlasStructType { } this.superTypes = Collections.unmodifiableList(s); - this.allSuperTypes = allS; + this.allSuperTypes = Collections.unmodifiableSet(allS); } public Set<String> getSuperTypes() { @@ -193,4 +184,41 @@ public class AtlasClassificationType extends AtlasStructType { super.populateDefaultValues(classification); } } + + private Set<String> getAllSuperTypes(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { + Set<String> superTypes = new HashSet<>(); + List<String> subTypes = new ArrayList<>(); + + collectAllSuperTypes(subTypes, superTypes, typeRegistry); + + return superTypes; + } + + /* + * This method should not assume that resolveReferences() has been called on all superTypes. + * this.classificationDef is the only safe member to reference here + */ + private void collectAllSuperTypes(List<String> subTypes, Set<String> superTypes, AtlasTypeRegistry typeRegistry) + throws AtlasBaseException { + if (subTypes.contains(classificationDef.getName())) { + throw new AtlasBaseException(classificationDef.getName() + + ": invalid supertypes - circular reference back to self " + subTypes); + } + + if (CollectionUtils.isNotEmpty(classificationDef.getSuperTypes())) { + superTypes.addAll(classificationDef.getSuperTypes()); + + subTypes.add(classificationDef.getName()); + for (String superTypeName : classificationDef.getSuperTypes()) { + AtlasType type = typeRegistry.getType(superTypeName); + + if (type instanceof AtlasClassificationType) { + AtlasClassificationType superType = (AtlasClassificationType) type; + + superType.collectAllSuperTypes(subTypes, superTypes, typeRegistry); + } + } + subTypes.remove(classificationDef.getName()); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5a4dd2e7/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java index 9ae61a7..bebfaeb 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java @@ -63,22 +63,13 @@ public class AtlasEntityType extends AtlasStructType { super.resolveReferences(typeRegistry); List<AtlasEntityType> s = new ArrayList<AtlasEntityType>(); - Set<String> allS = new HashSet<String>(); + Set<String> allS = getAllSuperTypes(typeRegistry); for (String superTypeName : entityDef.getSuperTypes()) { AtlasType superType = typeRegistry.getType(superTypeName); if (superType instanceof AtlasEntityType) { - AtlasEntityType superEntityType = (AtlasEntityType)superType; - - superEntityType.resolveReferences(typeRegistry); - - s.add(superEntityType); - allS.add(superTypeName); - - if (CollectionUtils.isNotEmpty(superEntityType.getAllSuperTypes())) { - allS.addAll(superEntityType.getAllSuperTypes()); - } + s.add((AtlasEntityType)superType); } else { throw new AtlasBaseException(superTypeName + ": incompatible supertype in entity " + entityDef.getName()); @@ -193,4 +184,41 @@ public class AtlasEntityType extends AtlasStructType { super.populateDefaultValues(ent); } } + + private Set<String> getAllSuperTypes(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { + Set<String> superTypes = new HashSet<>(); + List<String> subTypes = new ArrayList<>(); + + collectAllSuperTypes(subTypes, superTypes, typeRegistry); + + return superTypes; + } + + /* + * This method should not assume that resolveReferences() has been called on all superTypes. + * this.entityDef is the only safe member to reference here + */ + private void collectAllSuperTypes(List<String> subTypes, Set<String> superTypes, AtlasTypeRegistry typeRegistry) + throws AtlasBaseException { + if (subTypes.contains(entityDef.getName())) { + throw new AtlasBaseException(entityDef.getName() + + ": invalid supertypes - circular reference back to self " + subTypes); + } + + if (CollectionUtils.isNotEmpty(entityDef.getSuperTypes())) { + superTypes.addAll(entityDef.getSuperTypes()); + + subTypes.add(entityDef.getName()); + for (String superTypeName : entityDef.getSuperTypes()) { + AtlasType type = typeRegistry.getType(superTypeName); + + if (type instanceof AtlasEntityType) { + AtlasEntityType superType = (AtlasEntityType) type; + + superType.collectAllSuperTypes(subTypes, superTypes, typeRegistry); + } + } + subTypes.remove(entityDef.getName()); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5a4dd2e7/intg/src/test/java/org/apache/atlas/type/TestAtlasTypeRegistry.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/type/TestAtlasTypeRegistry.java b/intg/src/test/java/org/apache/atlas/type/TestAtlasTypeRegistry.java new file mode 100644 index 0000000..7ad9a18 --- /dev/null +++ b/intg/src/test/java/org/apache/atlas/type/TestAtlasTypeRegistry.java @@ -0,0 +1,270 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.type; + + +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.typedef.AtlasClassificationDef; +import org.apache.atlas.model.typedef.AtlasEntityDef; +import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.atlas.type.AtlasTypeRegistry.AtlasTransientTypeRegistry; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; + +public class TestAtlasTypeRegistry { + + /* + * L0 + * / \ + * / \ + * L1_1---- L1_2 + * / \ \ / \ + * / \ \ / \ + * L2_1 L2_2 L2_3 L2_4 + */ + @Test + public void testClassificationDefValidSuperTypes() { + AtlasClassificationDef classifiL0 = new AtlasClassificationDef("L0"); + AtlasClassificationDef classifiL1_1 = new AtlasClassificationDef("L1-1"); + AtlasClassificationDef classifiL1_2 = new AtlasClassificationDef("L1-2"); + AtlasClassificationDef classifiL2_1 = new AtlasClassificationDef("L2-1"); + AtlasClassificationDef classifiL2_2 = new AtlasClassificationDef("L2-2"); + AtlasClassificationDef classifiL2_3 = new AtlasClassificationDef("L2-3"); + AtlasClassificationDef classifiL2_4 = new AtlasClassificationDef("L2-4"); + + classifiL1_1.addSuperType(classifiL0.getName()); + classifiL1_2.addSuperType(classifiL0.getName()); + classifiL2_1.addSuperType(classifiL1_1.getName()); + classifiL2_2.addSuperType(classifiL1_1.getName()); + classifiL2_3.addSuperType(classifiL1_1.getName()); + classifiL2_3.addSuperType(classifiL1_2.getName()); + classifiL2_4.addSuperType(classifiL1_2.getName()); + + AtlasTypesDef typesDef = new AtlasTypesDef(); + + typesDef.getClassificationDefs().add(classifiL0); + typesDef.getClassificationDefs().add(classifiL1_1); + typesDef.getClassificationDefs().add(classifiL1_2); + typesDef.getClassificationDefs().add(classifiL2_1); + typesDef.getClassificationDefs().add(classifiL2_2); + typesDef.getClassificationDefs().add(classifiL2_3); + typesDef.getClassificationDefs().add(classifiL2_4); + + AtlasTypeRegistry typeRegistry = new AtlasTypeRegistry(); + AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry(); + String failureMsg = null; + + try { + ttr.addTypes(typesDef); + } catch (AtlasBaseException excp) { + failureMsg = excp.getMessage(); + } + assertNull(failureMsg); + } + + @Test + public void testClassificationDefInvalidSuperTypes_Self() { + AtlasClassificationDef classifiDef1 = new AtlasClassificationDef("classifiDef-1"); + + classifiDef1.addSuperType(classifiDef1.getName()); + + AtlasTypeRegistry typeRegistry = new AtlasTypeRegistry(); + AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry(); + String failureMsg = null; + + try { + ttr.addType(classifiDef1); + } catch (AtlasBaseException excp) { + failureMsg = excp.getMessage(); + } + assertNotNull(failureMsg, "expected invalid supertype failure"); + } + + /* + * L2_3 + * \ + * L0 + * / \ + * / \ + * L1_1---- L1_2 + * / \ \ / \ + * / \ \ / \ + * L2_1 L2_2 L2_3 L2_4 + */ + @Test + public void testClassificationDefInvalidSuperTypes_CircularRef() { + AtlasClassificationDef classifiL0 = new AtlasClassificationDef("L0"); + AtlasClassificationDef classifiL1_1 = new AtlasClassificationDef("L1-1"); + AtlasClassificationDef classifiL1_2 = new AtlasClassificationDef("L1-2"); + AtlasClassificationDef classifiL2_1 = new AtlasClassificationDef("L2-1"); + AtlasClassificationDef classifiL2_2 = new AtlasClassificationDef("L2-2"); + AtlasClassificationDef classifiL2_3 = new AtlasClassificationDef("L2-3"); + AtlasClassificationDef classifiL2_4 = new AtlasClassificationDef("L2-4"); + + classifiL1_1.addSuperType(classifiL0.getName()); + classifiL1_2.addSuperType(classifiL0.getName()); + classifiL2_1.addSuperType(classifiL1_1.getName()); + classifiL2_2.addSuperType(classifiL1_1.getName()); + classifiL2_3.addSuperType(classifiL1_1.getName()); + classifiL2_3.addSuperType(classifiL1_2.getName()); + classifiL2_4.addSuperType(classifiL1_2.getName()); + classifiL0.addSuperType(classifiL2_3.getName()); // circular-ref + + AtlasTypesDef typesDef = new AtlasTypesDef(); + + typesDef.getClassificationDefs().add(classifiL0); + typesDef.getClassificationDefs().add(classifiL1_1); + typesDef.getClassificationDefs().add(classifiL1_2); + typesDef.getClassificationDefs().add(classifiL2_1); + typesDef.getClassificationDefs().add(classifiL2_2); + typesDef.getClassificationDefs().add(classifiL2_3); + typesDef.getClassificationDefs().add(classifiL2_4); + + AtlasTypeRegistry typeRegistry = new AtlasTypeRegistry(); + AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry(); + String failureMsg = null; + + try { + ttr.addTypes(typesDef); + } catch (AtlasBaseException excp) { + failureMsg = excp.getMessage(); + } + assertNotNull(failureMsg, "expected invalid supertype failure"); + } + + /* + * L0 + * / \ + * / \ + * L1_1---- L1_2 + * / \ \ / \ + * / \ \ / \ + * L2_1 L2_2 L2_3 L2_4 + */ + @Test + public void testEntityDefValidSuperTypes() { + AtlasEntityDef entL0 = new AtlasEntityDef("L0"); + AtlasEntityDef entL1_1 = new AtlasEntityDef("L1-1"); + AtlasEntityDef entL1_2 = new AtlasEntityDef("L1-2"); + AtlasEntityDef entL2_1 = new AtlasEntityDef("L2-1"); + AtlasEntityDef entL2_2 = new AtlasEntityDef("L2-2"); + AtlasEntityDef entL2_3 = new AtlasEntityDef("L2-3"); + AtlasEntityDef entL2_4 = new AtlasEntityDef("L2-4"); + + entL1_1.addSuperType(entL0.getName()); + entL1_2.addSuperType(entL0.getName()); + entL2_1.addSuperType(entL1_1.getName()); + entL2_2.addSuperType(entL1_1.getName()); + entL2_3.addSuperType(entL1_1.getName()); + entL2_3.addSuperType(entL1_2.getName()); + entL2_4.addSuperType(entL1_2.getName()); + + AtlasTypesDef typesDef = new AtlasTypesDef(); + + typesDef.getEntityDefs().add(entL0); + typesDef.getEntityDefs().add(entL1_1); + typesDef.getEntityDefs().add(entL1_2); + typesDef.getEntityDefs().add(entL2_1); + typesDef.getEntityDefs().add(entL2_2); + typesDef.getEntityDefs().add(entL2_3); + typesDef.getEntityDefs().add(entL2_4); + + AtlasTypeRegistry typeRegistry = new AtlasTypeRegistry(); + AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry(); + String failureMsg = null; + + try { + ttr.addTypes(typesDef); + } catch (AtlasBaseException excp) { + failureMsg = excp.getMessage(); + } + assertNull(failureMsg); + } + + @Test + public void testEntityDefInvalidSuperTypes_Self() { + AtlasEntityDef entDef1 = new AtlasEntityDef("entDef-1"); + + entDef1.addSuperType(entDef1.getName()); + + AtlasTypeRegistry typeRegistry = new AtlasTypeRegistry(); + AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry(); + String failureMsg = null; + + try { + ttr.addType(entDef1); + } catch (AtlasBaseException excp) { + failureMsg = excp.getMessage(); + } + assertNotNull(failureMsg, "expected invalid supertype failure"); + } + + /* + * L2_3 + * \ + * L0 + * / \ + * / \ + * L1_1---- L1_2 + * / \ \ / \ + * / \ \ / \ + * L2_1 L2_2 L2_3 L2_4 + */ + @Test + public void testEntityDefInvalidSuperTypes_CircularRef() { + AtlasEntityDef entL0 = new AtlasEntityDef("L0"); + AtlasEntityDef entL1_1 = new AtlasEntityDef("L1-1"); + AtlasEntityDef entL1_2 = new AtlasEntityDef("L1-2"); + AtlasEntityDef entL2_1 = new AtlasEntityDef("L2-1"); + AtlasEntityDef entL2_2 = new AtlasEntityDef("L2-2"); + AtlasEntityDef entL2_3 = new AtlasEntityDef("L2-3"); + AtlasEntityDef entL2_4 = new AtlasEntityDef("L2-4"); + + entL1_1.addSuperType(entL0.getName()); + entL1_2.addSuperType(entL0.getName()); + entL2_1.addSuperType(entL1_1.getName()); + entL2_2.addSuperType(entL1_1.getName()); + entL2_3.addSuperType(entL1_1.getName()); + entL2_3.addSuperType(entL1_2.getName()); + entL2_4.addSuperType(entL1_2.getName()); + entL0.addSuperType(entL2_3.getName()); // circular-ref + + AtlasTypesDef typesDef = new AtlasTypesDef(); + + typesDef.getEntityDefs().add(entL0); + typesDef.getEntityDefs().add(entL1_1); + typesDef.getEntityDefs().add(entL1_2); + typesDef.getEntityDefs().add(entL2_1); + typesDef.getEntityDefs().add(entL2_2); + typesDef.getEntityDefs().add(entL2_3); + typesDef.getEntityDefs().add(entL2_4); + + AtlasTypeRegistry typeRegistry = new AtlasTypeRegistry(); + AtlasTransientTypeRegistry ttr = typeRegistry.createTransientTypeRegistry(); + String failureMsg = null; + + try { + ttr.addTypes(typesDef); + } catch (AtlasBaseException excp) { + failureMsg = excp.getMessage(); + } + assertNotNull(failureMsg, "expected invalid supertype failure"); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5a4dd2e7/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasClassificationDefStoreV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasClassificationDefStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasClassificationDefStoreV1.java index 4bdcf00..1794278 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasClassificationDefStoreV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasClassificationDefStoreV1.java @@ -173,8 +173,8 @@ public class AtlasClassificationDefStoreV1 extends AtlasAbstractDefStoreV1 imple LOG.debug("==> AtlasClassificationDefStoreV1.update({})", classifiDef); } - AtlasClassificationDef ret = StringUtils.isNotBlank(classifiDef.getName()) - ? updateByName(classifiDef.getName(), classifiDef) : updateByGuid(classifiDef.getGuid(), classifiDef); + AtlasClassificationDef ret = StringUtils.isNotBlank(classifiDef.getGuid()) + ? updateByGuid(classifiDef.getGuid(), classifiDef) : updateByName(classifiDef.getName(), classifiDef); if (LOG.isDebugEnabled()) { LOG.debug("<== AtlasClassificationDefStoreV1.update({}): {}", classifiDef, ret); @@ -360,13 +360,13 @@ public class AtlasClassificationDefStoreV1 extends AtlasAbstractDefStoreV1 imple private void updateVertexPreCreate(AtlasClassificationDef classificationDef, AtlasClassificationType classificationType, AtlasVertex vertex) { - AtlasStructDefStoreV1.updateVertexPreCreate(classificationDef, classificationType, vertex); + AtlasStructDefStoreV1.updateVertexPreCreate(classificationDef, classificationType, vertex, typeDefStore); } private void updateVertexPreUpdate(AtlasClassificationDef classificationDef, AtlasClassificationType classificationType, - AtlasVertex vertex) { - AtlasStructDefStoreV1.updateVertexPreUpdate(classificationDef, classificationType, vertex); + AtlasVertex vertex) throws AtlasBaseException { + AtlasStructDefStoreV1.updateVertexPreUpdate(classificationDef, classificationType, vertex, typeDefStore); } private void updateVertexAddReferences(AtlasClassificationDef classificationDef, AtlasVertex vertex) throws AtlasBaseException { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5a4dd2e7/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityDefStoreV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityDefStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityDefStoreV1.java index fd16cbd..a421006 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityDefStoreV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityDefStoreV1.java @@ -172,8 +172,8 @@ public class AtlasEntityDefStoreV1 extends AtlasAbstractDefStoreV1 implements At LOG.debug("==> AtlasEntityDefStoreV1.update({})", entityDef); } - AtlasEntityDef ret = StringUtils.isNotBlank(entityDef.getName()) ? updateByName(entityDef.getName(), entityDef) - : updateByGuid(entityDef.getGuid(), entityDef); + AtlasEntityDef ret = StringUtils.isNotBlank(entityDef.getGuid()) ? updateByGuid(entityDef.getGuid(), entityDef) + : updateByName(entityDef.getName(), entityDef); if (LOG.isDebugEnabled()) { LOG.debug("<== AtlasEntityDefStoreV1.update({}): {}", entityDef, ret); @@ -356,11 +356,12 @@ public class AtlasEntityDefStoreV1 extends AtlasAbstractDefStoreV1 implements At } private void updateVertexPreCreate(AtlasEntityDef entityDef, AtlasEntityType entityType, AtlasVertex vertex) { - AtlasStructDefStoreV1.updateVertexPreCreate(entityDef, entityType, vertex); + AtlasStructDefStoreV1.updateVertexPreCreate(entityDef, entityType, vertex, typeDefStore); } - private void updateVertexPreUpdate(AtlasEntityDef entityDef, AtlasEntityType entityType, AtlasVertex vertex) { - AtlasStructDefStoreV1.updateVertexPreUpdate(entityDef, entityType, vertex); + private void updateVertexPreUpdate(AtlasEntityDef entityDef, AtlasEntityType entityType, AtlasVertex vertex) + throws AtlasBaseException { + AtlasStructDefStoreV1.updateVertexPreUpdate(entityDef, entityType, vertex, typeDefStore); } private void updateVertexAddReferences(AtlasEntityDef entityDef, AtlasVertex vertex) throws AtlasBaseException { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5a4dd2e7/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java index d5fb584..a5c9093 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java @@ -81,7 +81,7 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At ret = typeDefStore.createTypeVertex(structDef); - AtlasStructDefStoreV1.updateVertexPreCreate(structDef, (AtlasStructType)type, ret); + AtlasStructDefStoreV1.updateVertexPreCreate(structDef, (AtlasStructType)type, ret, typeDefStore); if (LOG.isDebugEnabled()) { LOG.debug("<== AtlasStructDefStoreV1.preCreate({}): {}", structDef, ret); @@ -184,8 +184,8 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At LOG.debug("==> AtlasStructDefStoreV1.update({})", structDef); } - AtlasStructDef ret = StringUtils.isNotBlank(structDef.getName()) ? updateByName(structDef.getName(), structDef) - : updateByGuid(structDef.getGuid(), structDef); + AtlasStructDef ret = StringUtils.isNotBlank(structDef.getGuid()) ? updateByGuid(structDef.getGuid(), structDef) + : updateByName(structDef.getName(), structDef); if (LOG.isDebugEnabled()) { LOG.debug("<== AtlasStructDefStoreV1.update({}): {}", structDef, ret); @@ -212,7 +212,7 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At throw new AtlasBaseException("no structDef exists with name " + name); } - AtlasStructDefStoreV1.updateVertexPreUpdate(structDef, (AtlasStructType)type, vertex); + AtlasStructDefStoreV1.updateVertexPreUpdate(structDef, (AtlasStructType)type, vertex, typeDefStore); AtlasStructDefStoreV1.updateVertexAddReferences(structDef, vertex, typeDefStore); AtlasStructDef ret = toStructDef(vertex); @@ -242,7 +242,7 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At throw new AtlasBaseException("no structDef exists with guid " + guid); } - AtlasStructDefStoreV1.updateVertexPreUpdate(structDef, (AtlasStructType)type, vertex); + AtlasStructDefStoreV1.updateVertexPreUpdate(structDef, (AtlasStructType)type, vertex, typeDefStore); AtlasStructDefStoreV1.updateVertexAddReferences(structDef, vertex, typeDefStore); AtlasStructDef ret = toStructDef(vertex); @@ -377,7 +377,8 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At return ret; } - public static void updateVertexPreCreate(AtlasStructDef structDef, AtlasStructType structType, AtlasVertex vertex) { + public static void updateVertexPreCreate(AtlasStructDef structDef, AtlasStructType structType, + AtlasVertex vertex, AtlasTypeDefGraphStoreV1 typeDefStore) { List<String> attrNames = new ArrayList<>(structDef.getAttributeDefs().size()); for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) { @@ -390,8 +391,49 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At AtlasGraphUtilsV1.setProperty(vertex, AtlasGraphUtilsV1.getPropertyKey(structDef), attrNames); } - public static void updateVertexPreUpdate(AtlasStructDef structDef, AtlasStructType structType, AtlasVertex vertex) { - AtlasStructDefStoreV1.updateVertexPreCreate(structDef, structType, vertex); + public static void updateVertexPreUpdate(AtlasStructDef structDef, AtlasStructType structType, + AtlasVertex vertex, AtlasTypeDefGraphStoreV1 typeDefStore) + throws AtlasBaseException { + + List<String> attrNames = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(structDef.getAttributeDefs())) { + for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) { + attrNames.add(attributeDef.getName()); + } + } + + List<String> currAttrNames = vertex.getProperty(AtlasGraphUtilsV1.getPropertyKey(structDef), List.class); + + // delete attributes that are not present in updated structDef + if (CollectionUtils.isNotEmpty(currAttrNames)) { + for (String currAttrName : currAttrNames) { + if (!attrNames.contains(currAttrName)) { + throw new AtlasBaseException(structDef.getName() + "." + currAttrName + + ": attribute delete not supported"); + } + } + } + + typeDefStore.updateTypeVertex(structDef, vertex); + + // add/update attributes that are present in updated structDef + if (CollectionUtils.isNotEmpty(structDef.getAttributeDefs())) { + for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) { + if (CollectionUtils.isEmpty(currAttrNames) || !currAttrNames.contains(attributeDef.getName())) { + // new attribute - only allow if optional + if (!attributeDef.isOptional()) { + throw new AtlasBaseException(structDef.getName() + "." + attributeDef.getName() + + ": can not add mandatory attribute"); + } + } + + String propertyKey = AtlasGraphUtilsV1.getPropertyKey(structDef, attributeDef.getName()); + + AtlasGraphUtilsV1.setProperty(vertex, propertyKey, toJsonFromAttributeDef(attributeDef, structType)); + } + } + + AtlasGraphUtilsV1.setProperty(vertex, AtlasGraphUtilsV1.getPropertyKey(structDef), attrNames); } public static void updateVertexAddReferences(AtlasStructDef structDef, AtlasVertex vertex, http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5a4dd2e7/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java index 5e42c13..5131906 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java @@ -162,8 +162,6 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore { public AtlasVertex createTypeVertex(AtlasBaseTypeDef typeDef) { // Validate all the required checks Preconditions.checkArgument(StringUtils.isNotBlank(typeDef.getName()), "Type name can't be null/empty"); - Preconditions.checkArgument(StringUtils.isNotBlank(typeDef.getTypeVersion()), "Type version can't be null/empty"); - Preconditions.checkArgument(typeDef.getVersion() != null, "Version can't be null"); AtlasVertex ret = atlasGraph.addVertex(); @@ -171,6 +169,10 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore { typeDef.setTypeVersion("1.0"); } + if (typeDef.getVersion() == null) { + typeDef.setVersion(Long.valueOf(1l)); + } + if (StringUtils.isBlank(typeDef.getGuid())) { typeDef.setGuid(UUID.randomUUID().toString()); } @@ -198,6 +200,26 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore { return ret; } + public void updateTypeVertex(AtlasBaseTypeDef typeDef, AtlasVertex vertex) { + if (!isTypeVertex(vertex)) { + LOG.warn("updateTypeVertex(): not a type-vertex - {}", vertex); + + return; + } + + updateVertexProperty(vertex, Constants.GUID_PROPERTY_KEY, typeDef.getGuid()); + /* + * rename of a type is supported yet - as the typename is used to in the name of the edges from this vertex + * To support rename of types, he edge names should be derived from an internal-name - not directly the typename + * + updateVertexProperty(vertex, Constants.TYPENAME_PROPERTY_KEY, typeDef.getName()); + */ + updateVertexProperty(vertex, Constants.TYPEDESCRIPTION_PROPERTY_KEY, typeDef.getDescription()); + updateVertexProperty(vertex, Constants.TYPEVERSION_PROPERTY_KEY, typeDef.getTypeVersion()); + + markVertexUpdated(vertex); + } + public void deleteTypeVertexOutEdges(AtlasVertex vertex) throws AtlasBaseException { Iterable<AtlasEdge> edges = vertex.getEdges(AtlasEdgeDirection.OUT); @@ -311,15 +333,23 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore { return ret; } - public void createSuperTypeEdges(AtlasVertex vertex, Set<String> superTypes, TypeCategory typeCategory) { + public void createSuperTypeEdges(AtlasVertex vertex, Set<String> superTypes, TypeCategory typeCategory) + throws AtlasBaseException { + Set<String> currentSuperTypes = getSuperTypeNames(vertex); + if (CollectionUtils.isNotEmpty(superTypes)) { + if (! superTypes.containsAll(currentSuperTypes)) { + throw new AtlasBaseException("superType remove not supported"); + } + for (String superType : superTypes) { AtlasVertex superTypeVertex = findTypeVertexByNameAndCategory(superType, typeCategory); getOrCreateEdge(vertex, superTypeVertex, AtlasGraphUtilsV1.SUPERTYPE_EDGE_LABEL); } + } else if (CollectionUtils.isNotEmpty(currentSuperTypes)) { + throw new AtlasBaseException("superType remove not supported"); } - // TODO: remove any other superType edges, if any exists } public Set<String> getSuperTypeNames(AtlasVertex vertex) { @@ -348,4 +378,42 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore { return ret; } + + /* + * update the given vertex property, if the new value is not-blank + */ + private void updateVertexProperty(AtlasVertex vertex, String propertyName, String newValue) { + if (StringUtils.isNotBlank(newValue)) { + String currValue = vertex.getProperty(propertyName, String.class); + + if (!StringUtils.equals(currValue, newValue)) { + vertex.setProperty(propertyName, newValue); + } + } + } + + /* + * update the given vertex property, if the new value is not-null + */ + private void updateVertexProperty(AtlasVertex vertex, String propertyName, Date newValue) { + if (newValue != null) { + Number currValue = vertex.getProperty(propertyName, Number.class); + + if (currValue == null || !currValue.equals(newValue.getTime())) { + vertex.setProperty(propertyName, newValue.getTime()); + } + } + } + + /* + * increment the version value for this vertex + */ + private void markVertexUpdated(AtlasVertex vertex) { + Date now = new Date(); + Number currVersion = vertex.getProperty(Constants.VERSION_PROPERTY_KEY, Number.class); + long newVersion = currVersion == null ? 1 : (currVersion.longValue() + 1); + + vertex.setProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, now.getTime()); + vertex.setProperty(Constants.VERSION_PROPERTY_KEY, newVersion); + } }
