ATLAS-474 Server does not start if the type is updated with same super type class information (dkantor via shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/3cdd9f57 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/3cdd9f57 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/3cdd9f57 Branch: refs/heads/master Commit: 3cdd9f57d8b3cb31e407f2ac237676f34a7d08af Parents: 046ff52 Author: Shwetha GS <[email protected]> Authored: Fri Mar 11 11:08:06 2016 +0530 Committer: Shwetha GS <[email protected]> Committed: Fri Mar 11 11:08:06 2016 +0530 ---------------------------------------------------------------------- .../falcon/model/FalconDataModelGenerator.java | 4 +- .../hive/model/HiveDataModelGenerator.java | 14 ++-- .../sqoop/model/SqoopDataModelGenerator.java | 6 +- release-log.txt | 1 + .../typestore/GraphBackedTypeStore.java | 34 ++++++--- .../atlas/services/DefaultMetadataService.java | 9 ++- .../apache/atlas/BaseHiveRepositoryTest.java | 7 +- .../test/java/org/apache/atlas/TestUtils.java | 28 +++---- .../GraphBackedDiscoveryServiceTest.java | 10 ++- ...kedMetadataRepositoryDeleteEntitiesTest.java | 21 +++--- .../GraphBackedMetadataRepositoryTest.java | 3 +- .../typestore/GraphBackedTypeStoreTest.java | 77 +++++++++++++++++++- .../service/DefaultMetadataServiceTest.java | 5 +- .../atlas/typesystem/types/ClassType.java | 5 +- .../typesystem/types/HierarchicalType.java | 22 +++--- .../types/HierarchicalTypeDefinition.java | 10 ++- .../atlas/typesystem/types/TraitType.java | 7 +- .../atlas/typesystem/types/TypeSystem.java | 18 ++--- .../atlas/typesystem/types/utils/TypesUtil.java | 13 ++-- .../typesystem/builders/TypesBuilder.scala | 6 +- .../typesystem/json/TypesSerialization.scala | 18 ++--- .../typesystem/json/SerializationJavaTest.java | 12 +-- .../apache/atlas/typesystem/types/BaseTest.java | 10 ++- .../atlas/typesystem/types/ClassTest.java | 7 +- .../apache/atlas/typesystem/types/EnumTest.java | 4 +- .../typesystem/types/HierarchicalTypeTest.java | 9 ++- .../atlas/typesystem/types/TraitTest.java | 16 ++-- .../typesystem/types/TypeInheritanceTest.java | 19 ++--- .../atlas/typesystem/types/TypeSystemTest.java | 43 ++++++----- .../atlas/typesystem/types/ValidationTest.java | 14 ++-- .../typesystem/json/SerializationTest.scala | 15 ++-- .../org/apache/atlas/examples/QuickStart.java | 7 +- .../notification/EntityNotificationIT.java | 6 +- .../atlas/web/resources/BaseResourceIT.java | 22 +++--- .../web/resources/EntityJerseyResourceIT.java | 17 +++-- .../MetadataDiscoveryJerseyResourceIT.java | 19 +++-- .../web/resources/TypesJerseyResourceIT.java | 17 +++-- 37 files changed, 345 insertions(+), 210 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3cdd9f57/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java index 0dbb5dc..2494675 100644 --- a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java +++ b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java @@ -19,6 +19,8 @@ package org.apache.atlas.falcon.model; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; import org.apache.atlas.addons.ModelDefinitionDump; @@ -117,7 +119,7 @@ public class FalconDataModelGenerator { HierarchicalTypeDefinition<ClassType> definition = new HierarchicalTypeDefinition<>(ClassType.class, FalconDataTypes.FALCON_PROCESS_ENTITY.getName(), null, - ImmutableList.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions); + ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions); classTypeDefinitions.put(FalconDataTypes.FALCON_PROCESS_ENTITY.getName(), definition); LOG.debug("Created definition for {}", FalconDataTypes.FALCON_PROCESS_ENTITY.getName()); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3cdd9f57/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java index 2ba5170..c123719 100755 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java @@ -19,6 +19,8 @@ package org.apache.atlas.hive.model; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; import org.apache.atlas.addons.ModelDefinitionDump; @@ -212,7 +214,7 @@ public class HiveDataModelGenerator { HierarchicalTypeDefinition<ClassType> definition = new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_STORAGEDESC.getName(), null, - ImmutableList.of(AtlasClient.REFERENCEABLE_SUPER_TYPE), attributeDefinitions); + ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE), attributeDefinitions); classTypeDefinitions.put(HiveDataTypes.HIVE_STORAGEDESC.getName(), definition); LOG.debug("Created definition for " + HiveDataTypes.HIVE_STORAGEDESC.getName()); } @@ -247,7 +249,7 @@ public class HiveDataModelGenerator { HierarchicalTypeDefinition<ClassType> definition = new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_DB.getName(), null, - ImmutableList.of(AtlasClient.REFERENCEABLE_SUPER_TYPE), attributeDefinitions); + ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE), attributeDefinitions); classTypeDefinitions.put(HiveDataTypes.HIVE_DB.getName(), definition); LOG.debug("Created definition for " + HiveDataTypes.HIVE_DB.getName()); } @@ -274,7 +276,7 @@ public class HiveDataModelGenerator { new AttributeDefinition(COMMENT, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),}; HierarchicalTypeDefinition<ClassType> definition = new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_COLUMN.getName(), null, - ImmutableList.of(AtlasClient.REFERENCEABLE_SUPER_TYPE), attributeDefinitions); + ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE), attributeDefinitions); classTypeDefinitions.put(HiveDataTypes.HIVE_COLUMN.getName(), definition); LOG.debug("Created definition for " + HiveDataTypes.HIVE_COLUMN.getName()); } @@ -295,7 +297,7 @@ public class HiveDataModelGenerator { new AttributeDefinition(HiveDataModelGenerator.PARAMETERS, STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null),}; HierarchicalTypeDefinition<ClassType> definition = new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_PARTITION.getName(), null, - ImmutableList.of(AtlasClient.REFERENCEABLE_SUPER_TYPE), attributeDefinitions); + ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE), attributeDefinitions); classTypeDefinitions.put(HiveDataTypes.HIVE_PARTITION.getName(), definition); LOG.debug("Created definition for " + HiveDataTypes.HIVE_PARTITION.getName()); } @@ -329,7 +331,7 @@ public class HiveDataModelGenerator { null),}; HierarchicalTypeDefinition<ClassType> definition = new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TABLE.getName(), null, - ImmutableList.of("DataSet"), attributeDefinitions); + ImmutableSet.of("DataSet"), attributeDefinitions); classTypeDefinitions.put(HiveDataTypes.HIVE_TABLE.getName(), definition); LOG.debug("Created definition for " + HiveDataTypes.HIVE_TABLE.getName()); } @@ -368,7 +370,7 @@ public class HiveDataModelGenerator { HierarchicalTypeDefinition<ClassType> definition = new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_PROCESS.getName(), null, - ImmutableList.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions); + ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions); classTypeDefinitions.put(HiveDataTypes.HIVE_PROCESS.getName(), definition); LOG.debug("Created definition for " + HiveDataTypes.HIVE_PROCESS.getName()); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3cdd9f57/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java ---------------------------------------------------------------------- diff --git a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java index 7101225..c3bdbfa 100644 --- a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java +++ b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java @@ -19,6 +19,8 @@ package org.apache.atlas.sqoop.model; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; import org.apache.atlas.addons.ModelDefinitionDump; @@ -131,7 +133,7 @@ public class SqoopDataModelGenerator { HierarchicalTypeDefinition<ClassType> definition = new HierarchicalTypeDefinition<>(ClassType.class, SqoopDataTypes.SQOOP_DBDATASTORE.getName(), null, - ImmutableList.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions); + ImmutableSet.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions); classTypeDefinitions.put(SqoopDataTypes.SQOOP_DBDATASTORE.getName(), definition); LOG.debug("Created definition for " + SqoopDataTypes.SQOOP_DBDATASTORE.getName()); } @@ -150,7 +152,7 @@ public class SqoopDataModelGenerator { HierarchicalTypeDefinition<ClassType> definition = new HierarchicalTypeDefinition<>(ClassType.class, SqoopDataTypes.SQOOP_PROCESS.getName(), null, - ImmutableList.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions); + ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions); classTypeDefinitions.put(SqoopDataTypes.SQOOP_PROCESS.getName(), definition); LOG.debug("Created definition for " + SqoopDataTypes.SQOOP_PROCESS.getName()); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3cdd9f57/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 272edaf..78711c7 100644 --- a/release-log.txt +++ b/release-log.txt @@ -11,6 +11,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ALL CHANGES: +ATLAS-474 Server does not start if the type is updated with same super type class information (dkantor via shwethags) ATLAS-555 Tag creation from UI fails due to missing description attribute (guptaneeru via shwethags) ATLAS-522 Support Alter table commands (sumasai via shwethags) ATLAS-512 Decouple currently integrating components from availability of Atlas service for raising metadata events ( yhemanth via sumasai) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3cdd9f57/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java index c175dd4..71e9028 100755 --- a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java @@ -19,15 +19,18 @@ package org.apache.atlas.repository.typestore; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import com.google.inject.Singleton; import com.thinkaurelius.titan.core.TitanGraph; import com.tinkerpop.blueprints.Direction; import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Vertex; + import org.apache.atlas.AtlasException; import org.apache.atlas.GraphTransaction; import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graph.GraphProvider; import org.apache.atlas.typesystem.TypesDef; import org.apache.atlas.typesystem.types.AttributeDefinition; @@ -51,8 +54,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Set; @Singleton public class GraphBackedTypeStore implements ITypeStore { @@ -83,7 +88,7 @@ public class GraphBackedTypeStore implements ITypeStore { case STRUCT: StructType structType = (StructType) dataType; storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(), dataType.getDescription(), - ImmutableList.copyOf(structType.infoToNameMap.keySet()), ImmutableList.<String>of()); + ImmutableList.copyOf(structType.infoToNameMap.keySet()), ImmutableSet.<String>of()); break; case TRAIT: @@ -123,14 +128,12 @@ public class GraphBackedTypeStore implements ITypeStore { return PROPERTY_PREFIX + parent + "." + child; } - private String getEdgeLabel(String parent, String child) { + String getEdgeLabel(String parent, String child) { return PROPERTY_PREFIX + "edge." + parent + "." + child; } - private void storeInGraph(TypeSystem typeSystem, DataTypes.TypeCategory category, String typeName, - String typeDescription, ImmutableList<AttributeInfo> attributes, - ImmutableList<String> superTypes) - throws AtlasException { + private void storeInGraph(TypeSystem typeSystem, DataTypes.TypeCategory category, String typeName, String typeDescription, + ImmutableList<AttributeInfo> attributes, ImmutableSet<String> superTypes) throws AtlasException { Vertex vertex = createVertex(category, typeName, typeDescription); List<String> attrNames = new ArrayList<>(); if (attributes != null) { @@ -203,6 +206,15 @@ public class GraphBackedTypeStore implements ITypeStore { } private void addEdge(Vertex fromVertex, Vertex toVertex, String label) { + Iterable<Edge> edges = GraphHelper.getOutGoingEdgesByLabel(fromVertex, label); + // ATLAS-474: Check if this type system edge already exists, to avoid duplicates. + for (Edge edge : edges) { + if (edge.getVertex(Direction.IN).equals(toVertex)) { + LOG.debug("Edge from {} to {} with label {} already exists", + toString(fromVertex), toString(toVertex), label); + return; + } + } LOG.debug("Adding edge from {} to {} with label {}", toString(fromVertex), toString(toVertex), label); titanGraph.addEdge(null, fromVertex, toVertex, label); } @@ -236,7 +248,7 @@ public class GraphBackedTypeStore implements ITypeStore { break; case CLASS: - ImmutableList<String> superTypes = getSuperTypes(vertex); + ImmutableSet<String> superTypes = getSuperTypes(vertex); attributes = getAttributes(vertex, typeName); classTypes.add(new HierarchicalTypeDefinition(ClassType.class, typeName, typeDescription, superTypes, attributes)); break; @@ -266,14 +278,14 @@ public class GraphBackedTypeStore implements ITypeStore { return new EnumTypeDefinition(typeName, typeDescription, enumValues.toArray(new EnumValue[enumValues.size()])); } - private ImmutableList<String> getSuperTypes(Vertex vertex) { - List<String> superTypes = new ArrayList<>(); + private ImmutableSet<String> getSuperTypes(Vertex vertex) { + Set<String> superTypes = new HashSet<>(); Iterator<Edge> edges = vertex.getEdges(Direction.OUT, SUPERTYPE_EDGE_LABEL).iterator(); while (edges.hasNext()) { Edge edge = edges.next(); superTypes.add((String) edge.getVertex(Direction.IN).getProperty(Constants.TYPENAME_PROPERTY_KEY)); } - return ImmutableList.copyOf(superTypes); + return ImmutableSet.copyOf(superTypes); } private AttributeDefinition[] getAttributes(Vertex vertex, String typeName) throws AtlasException { @@ -302,7 +314,7 @@ public class GraphBackedTypeStore implements ITypeStore { * @param typeName * @return vertex */ - private Vertex findVertex(DataTypes.TypeCategory category, String typeName) { + Vertex findVertex(DataTypes.TypeCategory category, String typeName) { LOG.debug("Finding vertex for {}.{}", category, typeName); Iterator results = titanGraph.query().has(Constants.TYPENAME_PROPERTY_KEY, typeName).vertices().iterator(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3cdd9f57/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java index fe132b5..e326f27 100755 --- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java +++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java @@ -20,6 +20,7 @@ package org.apache.atlas.services; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.inject.Provider; import org.apache.atlas.AtlasClient; @@ -136,17 +137,17 @@ public class DefaultMetadataService implements MetadataService { @InterfaceAudience.Private private void createSuperTypes() throws AtlasException { HierarchicalTypeDefinition<ClassType> infraType = TypesUtil - .createClassTypeDef(AtlasClient.INFRASTRUCTURE_SUPER_TYPE, ImmutableList.<String>of(), NAME_ATTRIBUTE, + .createClassTypeDef(AtlasClient.INFRASTRUCTURE_SUPER_TYPE, ImmutableSet.<String>of(), NAME_ATTRIBUTE, DESCRIPTION_ATTRIBUTE); createType(infraType); HierarchicalTypeDefinition<ClassType> datasetType = TypesUtil - .createClassTypeDef(AtlasClient.DATA_SET_SUPER_TYPE, ImmutableList.<String>of(), NAME_ATTRIBUTE, + .createClassTypeDef(AtlasClient.DATA_SET_SUPER_TYPE, ImmutableSet.<String>of(), NAME_ATTRIBUTE, DESCRIPTION_ATTRIBUTE); createType(datasetType); HierarchicalTypeDefinition<ClassType> processType = TypesUtil - .createClassTypeDef(AtlasClient.PROCESS_SUPER_TYPE, ImmutableList.<String>of(), NAME_ATTRIBUTE, + .createClassTypeDef(AtlasClient.PROCESS_SUPER_TYPE, ImmutableSet.<String>of(), NAME_ATTRIBUTE, DESCRIPTION_ATTRIBUTE, new AttributeDefinition("inputs", DataTypes.arrayTypeName(AtlasClient.DATA_SET_SUPER_TYPE), Multiplicity.OPTIONAL, false, null), @@ -155,7 +156,7 @@ public class DefaultMetadataService implements MetadataService { createType(processType); HierarchicalTypeDefinition<ClassType> referenceableType = TypesUtil - .createClassTypeDef(AtlasClient.REFERENCEABLE_SUPER_TYPE, ImmutableList.<String>of(), + .createClassTypeDef(AtlasClient.REFERENCEABLE_SUPER_TYPE, ImmutableSet.<String>of(), TypesUtil.createUniqueRequiredAttrDef(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, DataTypes.STRING_TYPE)); createType(referenceableType); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3cdd9f57/repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java b/repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java index ea80314..d07f89d 100644 --- a/repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java +++ b/repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java @@ -19,8 +19,10 @@ package org.apache.atlas; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.util.TitanCleanup; + import org.apache.atlas.repository.MetadataRepository; import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; import org.apache.atlas.repository.graph.GraphProvider; @@ -44,6 +46,7 @@ import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.testng.annotations.Guice; import javax.inject.Inject; + import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -116,7 +119,7 @@ public class BaseHiveRepositoryTest { HierarchicalTypeDefinition<ClassType> tblClsDef = TypesUtil - .createClassTypeDef(HIVE_TABLE_TYPE, ImmutableList.of("DataSet"), + .createClassTypeDef(HIVE_TABLE_TYPE, ImmutableSet.of("DataSet"), attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.DATE_TYPE), attrDef("lastAccessTime", DataTypes.LONG_TYPE), attrDef("tableType", DataTypes.STRING_TYPE), @@ -129,7 +132,7 @@ public class BaseHiveRepositoryTest { Multiplicity.COLLECTION, true, null)); HierarchicalTypeDefinition<ClassType> loadProcessClsDef = TypesUtil - .createClassTypeDef(HIVE_PROCESS_TYPE, ImmutableList.of("Process"), + .createClassTypeDef(HIVE_PROCESS_TYPE, ImmutableSet.of("Process"), attrDef("userName", DataTypes.STRING_TYPE), attrDef("startTime", DataTypes.LONG_TYPE), attrDef("endTime", DataTypes.LONG_TYPE), attrDef("queryText", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3cdd9f57/repository/src/test/java/org/apache/atlas/TestUtils.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/TestUtils.java b/repository/src/test/java/org/apache/atlas/TestUtils.java index 8c6a6a1..7ca4c07 100755 --- a/repository/src/test/java/org/apache/atlas/TestUtils.java +++ b/repository/src/test/java/org/apache/atlas/TestUtils.java @@ -19,10 +19,12 @@ package org.apache.atlas; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.thinkaurelius.titan.core.TitanGraph; import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter; + import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.Referenceable; @@ -104,12 +106,12 @@ public final class TestUtils { createStructTypeDef("Address", "Address"+_description, createRequiredAttrDef("street", DataTypes.STRING_TYPE), createRequiredAttrDef("city", DataTypes.STRING_TYPE)); - HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department", "Department"+_description, ImmutableList.<String>of(), + HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department", "Department"+_description, ImmutableSet.<String>of(), createRequiredAttrDef("name", DataTypes.STRING_TYPE), new AttributeDefinition("employees", String.format("array<%s>", "Person"), Multiplicity.COLLECTION, true, "department")); - HierarchicalTypeDefinition<ClassType> personTypeDef = createClassTypeDef("Person", "Person"+_description, ImmutableList.<String>of(), + HierarchicalTypeDefinition<ClassType> personTypeDef = createClassTypeDef("Person", "Person"+_description, ImmutableSet.<String>of(), createRequiredAttrDef("name", DataTypes.STRING_TYPE), createOptionalAttrDef("orgLevel", "OrgLevel"), createOptionalAttrDef("address", "Address"), @@ -117,12 +119,12 @@ public final class TestUtils { new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates"), new AttributeDefinition("mentor", "Person", Multiplicity.OPTIONAL, false, null)); - HierarchicalTypeDefinition<ClassType> managerTypeDef = createClassTypeDef("Manager", "Manager"+_description, ImmutableList.of("Person"), + HierarchicalTypeDefinition<ClassType> managerTypeDef = createClassTypeDef("Manager", "Manager"+_description, ImmutableSet.of("Person"), new AttributeDefinition("subordinates", String.format("array<%s>", "Person"), Multiplicity.COLLECTION, false, "manager")); HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef = - createTraitTypeDef("SecurityClearance", "SecurityClearance"+_description, ImmutableList.<String>of(), + createTraitTypeDef("SecurityClearance", "SecurityClearance"+_description, ImmutableSet.<String>of(), createRequiredAttrDef("level", DataTypes.INT_TYPE)); ts.defineTypes(ImmutableList.of(orgLevelEnum), ImmutableList.of(addressDetails), @@ -201,13 +203,13 @@ public final class TestUtils { public static TypesDef defineHiveTypes() { String _description = "_description"; HierarchicalTypeDefinition<ClassType> superTypeDefinition = - createClassTypeDef(SUPER_TYPE_NAME, ImmutableList.<String>of(), + createClassTypeDef(SUPER_TYPE_NAME, ImmutableSet.<String>of(), createOptionalAttrDef("namespace", DataTypes.STRING_TYPE), createOptionalAttrDef("cluster", DataTypes.STRING_TYPE), createOptionalAttrDef("colo", DataTypes.STRING_TYPE)); HierarchicalTypeDefinition<ClassType> databaseTypeDefinition = - createClassTypeDef(DATABASE_TYPE, DATABASE_TYPE + _description,ImmutableList.of(SUPER_TYPE_NAME), + createClassTypeDef(DATABASE_TYPE, DATABASE_TYPE + _description,ImmutableSet.of(SUPER_TYPE_NAME), TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE), createOptionalAttrDef("created", DataTypes.DATE_TYPE), createRequiredAttrDef("description", DataTypes.STRING_TYPE)); @@ -223,7 +225,7 @@ public final class TestUtils { EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", "tableType" + _description, values); HierarchicalTypeDefinition<ClassType> columnsDefinition = - createClassTypeDef(COLUMN_TYPE, ImmutableList.<String>of(), + createClassTypeDef(COLUMN_TYPE, ImmutableSet.<String>of(), createRequiredAttrDef("name", DataTypes.STRING_TYPE), createRequiredAttrDef("type", DataTypes.STRING_TYPE)); @@ -245,7 +247,7 @@ public final class TestUtils { HierarchicalTypeDefinition<ClassType> storageDescClsDef = new HierarchicalTypeDefinition<>(ClassType.class, STORAGE_DESC_TYPE, STORAGE_DESC_TYPE + _description, - ImmutableList.of(SUPER_TYPE_NAME), attributeDefinitions); + ImmutableSet.of(SUPER_TYPE_NAME), attributeDefinitions); AttributeDefinition[] partClsAttributes = new AttributeDefinition[]{ new AttributeDefinition("values", DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName()), @@ -263,10 +265,10 @@ public final class TestUtils { HierarchicalTypeDefinition<ClassType> partClsDef = new HierarchicalTypeDefinition<>(ClassType.class, "partition_class_type", "partition_class_type" + _description, - ImmutableList.of(SUPER_TYPE_NAME), partClsAttributes); + ImmutableSet.of(SUPER_TYPE_NAME), partClsAttributes); HierarchicalTypeDefinition<ClassType> tableTypeDefinition = - createClassTypeDef(TABLE_TYPE, TABLE_TYPE + _description, ImmutableList.of(SUPER_TYPE_NAME), + createClassTypeDef(TABLE_TYPE, TABLE_TYPE + _description, ImmutableSet.of(SUPER_TYPE_NAME), TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE), createRequiredAttrDef("description", DataTypes.STRING_TYPE), createRequiredAttrDef("type", DataTypes.STRING_TYPE), @@ -306,14 +308,14 @@ public final class TestUtils { new AttributeDefinition("databaseComposite", DATABASE_TYPE, Multiplicity.OPTIONAL, true, null)); HierarchicalTypeDefinition<TraitType> piiTypeDefinition = - createTraitTypeDef(PII, PII + _description, ImmutableList.<String>of()); + createTraitTypeDef(PII, PII + _description, ImmutableSet.<String>of()); HierarchicalTypeDefinition<TraitType> classificationTypeDefinition = - createTraitTypeDef(CLASSIFICATION, CLASSIFICATION + _description, ImmutableList.<String>of(), + createTraitTypeDef(CLASSIFICATION, CLASSIFICATION + _description, ImmutableSet.<String>of(), createRequiredAttrDef("tag", DataTypes.STRING_TYPE)); HierarchicalTypeDefinition<TraitType> fetlClassificationTypeDefinition = - createTraitTypeDef("fetl" + CLASSIFICATION, "fetl" + CLASSIFICATION + _description, ImmutableList.of(CLASSIFICATION), + createTraitTypeDef("fetl" + CLASSIFICATION, "fetl" + CLASSIFICATION + _description, ImmutableSet.of(CLASSIFICATION), createRequiredAttrDef("tag", DataTypes.STRING_TYPE)); return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3cdd9f57/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java index 2400b03..5b74dc8 100755 --- a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java @@ -18,8 +18,9 @@ package org.apache.atlas.discovery; -import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.thinkaurelius.titan.core.TitanGraph; + import org.apache.atlas.BaseHiveRepositoryTest; import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.TestUtils; @@ -45,6 +46,7 @@ import org.testng.annotations.Guice; import org.testng.annotations.Test; import javax.inject.Inject; + import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -305,13 +307,13 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest { HierarchicalTypeDefinition A = createClassTypeDef("A", null, createRequiredAttrDef("a", DataTypes.INT_TYPE)); HierarchicalTypeDefinition B = - createClassTypeDef("B", ImmutableList.of("A"), createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE)); + createClassTypeDef("B", ImmutableSet.of("A"), createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE)); HierarchicalTypeDefinition C = - createClassTypeDef("C", ImmutableList.of("B"), createOptionalAttrDef("c", DataTypes.BYTE_TYPE)); + createClassTypeDef("C", ImmutableSet.of("B"), createOptionalAttrDef("c", DataTypes.BYTE_TYPE)); HierarchicalTypeDefinition D = - createClassTypeDef("D", ImmutableList.of("C"), createOptionalAttrDef("d", DataTypes.SHORT_TYPE)); + createClassTypeDef("D", ImmutableSet.of("C"), createOptionalAttrDef("d", DataTypes.SHORT_TYPE)); TypeSystem.getInstance().defineClassTypes(A, B, C, D); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3cdd9f57/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java index 3681de7..975d530 100644 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java @@ -18,10 +18,8 @@ package org.apache.atlas.repository.graph; -import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef; -import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef; - import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.util.TitanCleanup; import com.tinkerpop.blueprints.Vertex; @@ -61,7 +59,6 @@ import javax.inject.Inject; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Date; import java.util.List; import java.util.Map; @@ -163,12 +160,12 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest { public void testDeleteEntitiesWithCompositeMapReference() throws Exception { // Define type for map value. HierarchicalTypeDefinition<ClassType> mapValueDef = TypesUtil.createClassTypeDef("CompositeMapValue", - ImmutableList.<String>of(), + ImmutableSet.<String>of(), TypesUtil.createOptionalAttrDef("attr1", DataTypes.STRING_TYPE)); // Define type with map where the value is a composite class reference to MapValue. HierarchicalTypeDefinition<ClassType> mapOwnerDef = TypesUtil.createClassTypeDef("CompositeMapOwner", - ImmutableList.<String>of(), + ImmutableSet.<String>of(), new AttributeDefinition("map", DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), "CompositeMapValue"), Multiplicity.OPTIONAL, true, null)); TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), @@ -342,11 +339,11 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest { public void testDisconnectUnidirectionalArrayReferenceFromStructAndTraitTypes() throws Exception { // Define class types. HierarchicalTypeDefinition<ClassType> structTargetDef = TypesUtil.createClassTypeDef("StructTarget", - ImmutableList.<String>of(), TypesUtil.createOptionalAttrDef("attr1", DataTypes.STRING_TYPE)); + ImmutableSet.<String>of(), TypesUtil.createOptionalAttrDef("attr1", DataTypes.STRING_TYPE)); HierarchicalTypeDefinition<ClassType> traitTargetDef = TypesUtil.createClassTypeDef("TraitTarget", - ImmutableList.<String>of(), TypesUtil.createOptionalAttrDef("attr1", DataTypes.STRING_TYPE)); + ImmutableSet.<String>of(), TypesUtil.createOptionalAttrDef("attr1", DataTypes.STRING_TYPE)); HierarchicalTypeDefinition<ClassType> structContainerDef = TypesUtil.createClassTypeDef("StructContainer", - ImmutableList.<String>of(), TypesUtil.createOptionalAttrDef("struct", "TestStruct")); + ImmutableSet.<String>of(), TypesUtil.createOptionalAttrDef("struct", "TestStruct")); // Define struct and trait types which have a unidirectional array reference // to a class type. @@ -355,7 +352,7 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest { new AttributeDefinition("nestedStructs", DataTypes.arrayTypeName("NestedStruct"), Multiplicity.OPTIONAL, false, null)); StructTypeDefinition nestedStructDef = TypesUtil.createStructTypeDef("NestedStruct", TypesUtil.createOptionalAttrDef("attr1", DataTypes.STRING_TYPE)); - HierarchicalTypeDefinition<TraitType> traitDef = TypesUtil.createTraitTypeDef("TestTrait", ImmutableList.<String>of(), + HierarchicalTypeDefinition<TraitType> traitDef = TypesUtil.createTraitTypeDef("TestTrait", ImmutableSet.<String>of(), new AttributeDefinition("target", DataTypes.arrayTypeName("TraitTarget"), Multiplicity.OPTIONAL, false, null)); TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.of(structDef, nestedStructDef), @@ -476,13 +473,13 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest { public void testDisconnectMapReferenceFromClassType() throws Exception { // Define type for map value. HierarchicalTypeDefinition<ClassType> mapValueDef = TypesUtil.createClassTypeDef("MapValue", - ImmutableList.<String>of(), + ImmutableSet.<String>of(), new AttributeDefinition("biMapOwner", "MapOwner", Multiplicity.OPTIONAL, false, "biMap")); // Define type with unidirectional and bidirectional map references, // where the map value is a class reference to MapValue. HierarchicalTypeDefinition<ClassType> mapOwnerDef = TypesUtil.createClassTypeDef("MapOwner", - ImmutableList.<String>of(), + ImmutableSet.<String>of(), new AttributeDefinition("map", DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), "MapValue"), Multiplicity.OPTIONAL, false, null), new AttributeDefinition("biMap", DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3cdd9f57/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java index 83e4d85..f1dd604 100755 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java @@ -19,6 +19,7 @@ package org.apache.atlas.repository.graph; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.util.TitanCleanup; import com.tinkerpop.blueprints.Compare; @@ -274,7 +275,7 @@ public class GraphBackedMetadataRepositoryTest { final String traitName = "P_I_I"; HierarchicalTypeDefinition<TraitType> piiTrait = TypesUtil - .createTraitTypeDef(traitName, ImmutableList.<String>of(), + .createTraitTypeDef(traitName, ImmutableSet.<String>of(), TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE)); TraitType traitType = typeSystem.defineTraitType(piiTrait); ITypedStruct traitInstance = traitType.createInstance(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3cdd9f57/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java b/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java index 10f8ee3..8ca3132 100755 --- a/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java @@ -19,6 +19,7 @@ package org.apache.atlas.repository.typestore; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.util.TitanCleanup; import com.tinkerpop.blueprints.Direction; @@ -34,6 +35,7 @@ import org.apache.atlas.typesystem.TypesDef; import org.apache.atlas.typesystem.types.AttributeDefinition; import org.apache.atlas.typesystem.types.ClassType; import org.apache.atlas.typesystem.types.DataTypes; +import org.apache.atlas.typesystem.types.DataTypes.TypeCategory; import org.apache.atlas.typesystem.types.EnumType; import org.apache.atlas.typesystem.types.EnumTypeDefinition; import org.apache.atlas.typesystem.types.EnumValue; @@ -52,6 +54,9 @@ import org.testng.annotations.Guice; import org.testng.annotations.Test; import javax.inject.Inject; + +import java.util.Arrays; +import java.util.Iterator; import java.util.List; import java.util.Map; @@ -168,11 +173,11 @@ public class GraphBackedTypeStoreTest { createOptionalAttrDef("state", DataTypes.STRING_TYPE)); //Add supertype - HierarchicalTypeDefinition<ClassType> superTypeDef = createClassTypeDef("Division", ImmutableList.<String>of(), + HierarchicalTypeDefinition<ClassType> superTypeDef = createClassTypeDef("Division", ImmutableSet.<String>of(), createOptionalAttrDef("dname", DataTypes.STRING_TYPE)); HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department", "Department"+_description, - ImmutableList.of(superTypeDef.typeName), createRequiredAttrDef("name", DataTypes.STRING_TYPE), + ImmutableSet.of(superTypeDef.typeName), createRequiredAttrDef("name", DataTypes.STRING_TYPE), new AttributeDefinition("employees", String.format("array<%s>", "Person"), Multiplicity.COLLECTION, true, "department")); TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.of(orgLevelEnum), ImmutableList.of(addressDetails), @@ -182,6 +187,16 @@ public class GraphBackedTypeStoreTest { Map<String, IDataType> typesAdded = ts.updateTypes(typesDef); typeStore.store(ts, ImmutableList.copyOf(typesAdded.keySet())); + // ATLAS-474: verify that type update did not write duplicate edges to the type store. + if (typeStore instanceof GraphBackedTypeStore) { + GraphBackedTypeStore gbTypeStore = (GraphBackedTypeStore) typeStore; + Vertex typeVertex = gbTypeStore.findVertex(TypeCategory.CLASS, "Department"); + int edgeCount = countOutgoingEdges(typeVertex, GraphBackedTypeStore.SUPERTYPE_EDGE_LABEL); + Assert.assertEquals(edgeCount, 1); + edgeCount = countOutgoingEdges(typeVertex, gbTypeStore.getEdgeLabel("Department", "employees")); + Assert.assertEquals(edgeCount, 1, "Should only be 1 edge for employees attribute on Department type vertex"); + } + //Validate the updated types TypesDef types = typeStore.restore(); ts.reset(); @@ -204,4 +219,62 @@ public class GraphBackedTypeStoreTest { Assert.assertTrue(deptType.superTypes.contains(superTypeDef.typeName)); Assert.assertNotNull(ts.getDataType(ClassType.class, superTypeDef.typeName)); } + + @Test(dependsOnMethods = "testTypeUpdate") + public void testAddSecondSuperType() throws Exception { + // Add a second supertype to Department class + HierarchicalTypeDefinition<ClassType> superTypeDef2 = createClassTypeDef("SuperClass2", ImmutableSet.<String>of(), + createOptionalAttrDef("name", DataTypes.STRING_TYPE)); + HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department", + ImmutableSet.of("Division", superTypeDef2.typeName), createRequiredAttrDef("name", DataTypes.STRING_TYPE), + new AttributeDefinition("employees", String.format("array<%s>", "Person"), Multiplicity.COLLECTION, + true, "department")); + TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), + ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), + ImmutableList.of(deptTypeDef, superTypeDef2)); + ts.updateTypes(typesDef); + typeStore.store(ts, ImmutableList.of(superTypeDef2.typeName, deptTypeDef.typeName)); + + // ATLAS-474: verify that type update did not write duplicate edges to the type store. + if (typeStore instanceof GraphBackedTypeStore) { + GraphBackedTypeStore gbTypeStore = (GraphBackedTypeStore) typeStore; + Vertex typeVertex = gbTypeStore.findVertex(TypeCategory.CLASS, "Department"); + // There should now be 2 super type outgoing edges on the Department type vertex. + int edgeCount = countOutgoingEdges(typeVertex, GraphBackedTypeStore.SUPERTYPE_EDGE_LABEL); + Assert.assertEquals(edgeCount, 2); + // There should still be 1 outgoing edge for the employees attribute. + edgeCount = countOutgoingEdges(typeVertex, gbTypeStore.getEdgeLabel("Department", "employees")); + Assert.assertEquals(edgeCount, 1); + } + + // Verify Department now has 2 super types. + TypesDef types = typeStore.restore(); + for (HierarchicalTypeDefinition<ClassType> classTypeDef : types.classTypesAsJavaList()) { + if (classTypeDef.typeName.equals("Department")) { + Assert.assertEquals(classTypeDef.superTypes.size(), 2); + Assert.assertTrue(classTypeDef.superTypes.containsAll( + Arrays.asList("Division", superTypeDef2.typeName))); + break; + } + } + ts.reset(); + Map<String, IDataType> typesMap = ts.defineTypes(types); + IDataType dataType = typesMap.get(deptTypeDef.typeName); + Assert.assertTrue(dataType instanceof ClassType); + ClassType deptType = (ClassType) dataType; + Assert.assertEquals(deptType.superTypes.size(), 2); + Assert.assertTrue(deptType.superTypes.containsAll( + Arrays.asList("Division", superTypeDef2.typeName))); + } + + private int countOutgoingEdges(Vertex typeVertex, String edgeLabel) { + + Iterable<Edge> outGoingEdgesByLabel = GraphHelper.getOutGoingEdgesByLabel(typeVertex, edgeLabel); + int edgeCount = 0; + for (Iterator<Edge> iterator = outGoingEdgesByLabel.iterator(); iterator.hasNext();) { + iterator.next(); + edgeCount++; + } + return edgeCount; + } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3cdd9f57/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java index 0218ee5..0b01230 100644 --- a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java @@ -19,6 +19,7 @@ package org.apache.atlas.service; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.util.TitanCleanup; @@ -802,14 +803,14 @@ public class DefaultMetadataServiceTest { public void testTypeUpdateWithReservedAttributes() throws AtlasException, JSONException { String typeName = "test_type_"+ RandomStringUtils.randomAlphanumeric(10); HierarchicalTypeDefinition<ClassType> typeDef = TypesUtil.createClassTypeDef( - typeName, ImmutableList.<String>of(), + typeName, ImmutableSet.<String>of(), TypesUtil.createUniqueRequiredAttrDef("test_type_attribute", DataTypes.STRING_TYPE)); TypesDef typesDef = new TypesDef(typeDef, false); JSONObject type = metadataService.createType(TypesSerialization.toJson(typesDef)); Assert.assertNotNull(type.get(AtlasClient.TYPES)); HierarchicalTypeDefinition<ClassType> updatedTypeDef = TypesUtil.createClassTypeDef( - typeName, ImmutableList.<String>of(), + typeName, ImmutableSet.<String>of(), TypesUtil.createUniqueRequiredAttrDef("test_type_attribute", DataTypes.STRING_TYPE), TypesUtil.createOptionalAttrDef("test_type_invalid_attribute$", DataTypes.STRING_TYPE)); TypesDef updatedTypesDef = new TypesDef(updatedTypeDef, false); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3cdd9f57/typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java index d95e80a..475ac9f 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java @@ -21,6 +21,7 @@ package org.apache.atlas.typesystem.types; import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import org.apache.atlas.AtlasException; import org.apache.atlas.typesystem.IReferenceableInstance; @@ -48,12 +49,12 @@ public class ClassType extends HierarchicalType<ClassType, IReferenceableInstanc public final Map<AttributeInfo, List<String>> infoToNameMap; - ClassType(TypeSystem typeSystem, String name, String description, ImmutableList<String> superTypes, int numFields) { + ClassType(TypeSystem typeSystem, String name, String description, ImmutableSet<String> superTypes, int numFields) { super(typeSystem, ClassType.class, name, description, superTypes, numFields); infoToNameMap = null; } - ClassType(TypeSystem typeSystem, String name, String description, ImmutableList<String> superTypes, AttributeInfo... fields) + ClassType(TypeSystem typeSystem, String name, String description, ImmutableSet<String> superTypes, AttributeInfo... fields) throws AtlasException { super(typeSystem, ClassType.class, name, description, superTypes, fields); infoToNameMap = TypeUtils.buildAttrInfoToNameMap(fieldMapping); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3cdd9f57/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java index 7c8db84..ac32126 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java @@ -20,6 +20,8 @@ package org.apache.atlas.typesystem.types; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + import org.apache.atlas.AtlasException; import org.apache.atlas.typesystem.IStruct; import org.apache.atlas.typesystem.persistence.DownCastStructInstance; @@ -51,13 +53,13 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A public final Class<ST> superTypeClass; public final FieldMapping fieldMapping; public final int numFields; - public final ImmutableList<String> superTypes; + public final ImmutableSet<String> superTypes; public final ImmutableList<AttributeInfo> immediateAttrs; public final ImmutableMap<String, String> attributeNameToType; protected ImmutableMap<String, List<Path>> superTypePaths; protected ImmutableMap<String, Path> pathNameToPathMap; - HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, ImmutableList<String> superTypes, + HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, ImmutableSet<String> superTypes, int numFields) { this(typeSystem, superTypeClass, name, null, superTypes, numFields); } @@ -65,7 +67,7 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A /** * Used when creating a Type, to support recursive Structs. */ - HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, String description, ImmutableList<String> superTypes, + HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, String description, ImmutableSet<String> superTypes, int numFields) { super(name, description); this.typeSystem = typeSystem; @@ -77,11 +79,11 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A this.attributeNameToType = null; } - HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, ImmutableList<String> superTypes, + HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, ImmutableSet<String> superTypes, AttributeInfo... fields) throws AtlasException { this(typeSystem, superTypeClass, name, null, superTypes, fields); } - HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, String description, ImmutableList<String> superTypes, + HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, String description, ImmutableSet<String> superTypes, AttributeInfo... fields) throws AtlasException { super(name, description); this.typeSystem = typeSystem; @@ -90,7 +92,7 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A this.fieldMapping = p.left; this.attributeNameToType = p.right; this.numFields = this.fieldMapping.fields.size(); - this.superTypes = superTypes == null ? ImmutableList.<String>of() : superTypes; + this.superTypes = superTypes == null ? ImmutableSet.<String>of() : superTypes; this.immediateAttrs = ImmutableList.copyOf(fields); } @@ -136,7 +138,7 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A setupSuperTypesGraph(superTypes); } - private void setupSuperTypesGraph(ImmutableList<String> superTypes) throws AtlasException { + private void setupSuperTypesGraph(ImmutableSet<String> superTypes) throws AtlasException { Map<String, List<Path>> superTypePaths = new HashMap<String, List<Path>>(); Map<String, Path> pathNameToPathMap = new HashMap<String, Path>(); Queue<Path> queue = new LinkedList<Path>(); @@ -157,7 +159,7 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A typePaths.add(currentPath); } - ImmutableList<String> sTs = superType == this ? superTypes : superType.superTypes; + ImmutableSet<String> sTs = superType == this ? superTypes : superType.superTypes; if (sTs != null) { for (String sT : sTs) { @@ -171,7 +173,7 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A } - protected Pair<FieldMapping, ImmutableMap<String, String>> constructFieldMapping(ImmutableList<String> superTypes, + protected Pair<FieldMapping, ImmutableMap<String, String>> constructFieldMapping(ImmutableSet<String> superTypes, AttributeInfo... fields) throws AtlasException { Map<String, AttributeInfo> fieldsMap = new LinkedHashMap(); @@ -479,7 +481,7 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A throw new RuntimeException(me); } if (t.superTypes != null) { - ImmutableList<String> sTs = t.superTypes; + ImmutableSet<String> sTs = t.superTypes; for (String sT : sTs) { String nm = sT + "." + p.pathName; pathQueue.add(pathNameToPathMap.get(nm)); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3cdd9f57/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalTypeDefinition.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalTypeDefinition.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalTypeDefinition.java index 973a591..9a299f0 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalTypeDefinition.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalTypeDefinition.java @@ -19,11 +19,13 @@ package org.apache.atlas.typesystem.types; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + import org.apache.atlas.classification.InterfaceAudience; public class HierarchicalTypeDefinition<T extends HierarchicalType> extends StructTypeDefinition { - public final ImmutableList<String> superTypes; + public final ImmutableSet<String> superTypes; public final String hierarchicalMetaTypeName; /** @@ -39,15 +41,15 @@ public class HierarchicalTypeDefinition<T extends HierarchicalType> extends Stru @InterfaceAudience.Private public HierarchicalTypeDefinition(String hierarchicalMetaTypeName, String typeName, String typeDescription, String[] superTypes, AttributeDefinition[] attributeDefinitions) throws ClassNotFoundException { - this((Class<T>) Class.forName(hierarchicalMetaTypeName), typeName, typeDescription, ImmutableList.copyOf(superTypes), + this((Class<T>) Class.forName(hierarchicalMetaTypeName), typeName, typeDescription, ImmutableSet.copyOf(superTypes), attributeDefinitions); } - public HierarchicalTypeDefinition(Class<T> hierarchicalMetaType, String typeName, String typeDescription, ImmutableList<String> superTypes, + public HierarchicalTypeDefinition(Class<T> hierarchicalMetaType, String typeName, String typeDescription, ImmutableSet<String> superTypes, AttributeDefinition[] attributeDefinitions) { super(typeName, typeDescription, false, attributeDefinitions); hierarchicalMetaTypeName = hierarchicalMetaType.getName(); - this.superTypes = superTypes == null ? ImmutableList.<String>of() : superTypes; + this.superTypes = superTypes == null ? ImmutableSet.<String>of() : superTypes; } @Override http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3cdd9f57/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java index baf5b4d..84c22bf 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java @@ -18,7 +18,8 @@ package org.apache.atlas.typesystem.types; -import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + import org.apache.atlas.AtlasException; import org.apache.atlas.typesystem.IStruct; import org.apache.atlas.typesystem.ITypedStruct; @@ -34,13 +35,13 @@ public class TraitType extends HierarchicalType<TraitType, IStruct> public final Map<AttributeInfo, List<String>> infoToNameMap; private final TypedStructHandler handler; - TraitType(TypeSystem typeSystem, String name, String description, ImmutableList<String> superTraits, int numFields) { + TraitType(TypeSystem typeSystem, String name, String description, ImmutableSet<String> superTraits, int numFields) { super(typeSystem, TraitType.class, name, description, superTraits, numFields); handler = null; infoToNameMap = null; } - TraitType(TypeSystem typeSystem, String name, String description, ImmutableList<String> superTraits, AttributeInfo... fields) + TraitType(TypeSystem typeSystem, String name, String description, ImmutableSet<String> superTraits, AttributeInfo... fields) throws AtlasException { super(typeSystem, TraitType.class, name, description, superTraits, fields); handler = new TypedStructHandler(this); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3cdd9f57/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java index 3c479df..9e4aa79 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java @@ -20,7 +20,9 @@ package org.apache.atlas.typesystem.types; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Multimap; + import org.apache.atlas.AtlasException; import org.apache.atlas.classification.InterfaceAudience; import org.apache.atlas.typesystem.TypesDef; @@ -28,12 +30,12 @@ import org.apache.atlas.typesystem.exception.TypeExistsException; import org.apache.atlas.typesystem.exception.TypeNotFoundException; import javax.inject.Singleton; + import java.lang.reflect.Constructor; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -44,7 +46,7 @@ import java.util.concurrent.ConcurrentHashMap; @InterfaceAudience.Private public class TypeSystem { private static final TypeSystem INSTANCE = new TypeSystem(); - private static ThreadLocal<SimpleDateFormat> dateFormat = new ThreadLocal() { + private static ThreadLocal<SimpleDateFormat> dateFormat = new ThreadLocal<SimpleDateFormat>() { @Override public SimpleDateFormat initialValue() { SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); @@ -426,14 +428,7 @@ public class TypeSystem { private <U extends HierarchicalType> void validateSuperTypes(Class<U> cls, HierarchicalTypeDefinition<U> def) throws AtlasException { - Set<String> s = new HashSet<>(); - ImmutableList<String> superTypes = def.superTypes; - for (String superTypeName : superTypes) { - - if (s.contains(superTypeName)) { - throw new AtlasException( - String.format("Type %s extends superType %s multiple times", def.typeName, superTypeName)); - } + for (String superTypeName : def.superTypes) { IDataType dT = dataType(superTypeName); @@ -448,7 +443,6 @@ public class TypeSystem { String.format("SuperType %s must be a %s, in definition of type %s", superTypeName, cls.getName(), def.typeName)); } - s.add(superTypeName); } } @@ -520,7 +514,7 @@ public class TypeSystem { } try { - Constructor<U> cons = cls.getDeclaredConstructor(TypeSystem.class, String.class, String.class, ImmutableList.class, + Constructor<U> cons = cls.getDeclaredConstructor(TypeSystem.class, String.class, String.class, ImmutableSet.class, AttributeInfo[].class); U type = cons.newInstance(this, def.typeName, def.typeDescription, def.superTypes, infos); transientTypes.put(def.typeName, type); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3cdd9f57/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java index 80968d8..2ea90a3 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java @@ -19,6 +19,8 @@ package org.apache.atlas.typesystem.types.utils; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + import org.apache.atlas.typesystem.TypesDef; import org.apache.atlas.typesystem.types.AttributeDefinition; import org.apache.atlas.typesystem.types.ClassType; @@ -29,6 +31,7 @@ import org.apache.atlas.typesystem.types.IDataType; import org.apache.atlas.typesystem.types.Multiplicity; import org.apache.atlas.typesystem.types.StructTypeDefinition; import org.apache.atlas.typesystem.types.TraitType; + import scala.collection.JavaConversions; /** @@ -64,12 +67,12 @@ public class TypesUtil { } public static HierarchicalTypeDefinition<TraitType> createTraitTypeDef(String name, - ImmutableList<String> superTypes, AttributeDefinition... attrDefs) { + ImmutableSet<String> superTypes, AttributeDefinition... attrDefs) { return createTraitTypeDef(name, null, superTypes, attrDefs); } public static HierarchicalTypeDefinition<TraitType> createTraitTypeDef(String name, String description, - ImmutableList<String> superTypes, AttributeDefinition... attrDefs) { + ImmutableSet<String> superTypes, AttributeDefinition... attrDefs) { return new HierarchicalTypeDefinition<>(TraitType.class, name, description, superTypes, attrDefs); } @@ -82,13 +85,13 @@ public class TypesUtil { } public static HierarchicalTypeDefinition<ClassType> createClassTypeDef(String name, - ImmutableList<String> superTypes, AttributeDefinition... attrDefs) { + ImmutableSet<String> superTypes, AttributeDefinition... attrDefs) { return createClassTypeDef(name, null, superTypes, attrDefs); } public static HierarchicalTypeDefinition<ClassType> createClassTypeDef(String name, String description, - ImmutableList<String> superTypes, AttributeDefinition... attrDefs) { - return new HierarchicalTypeDefinition<>(ClassType.class, name, description, superTypes, attrDefs); + ImmutableSet<String> superTypes, AttributeDefinition... attrDefs) { + return new HierarchicalTypeDefinition<>(ClassType.class, name, description, superTypes, attrDefs); } public static TypesDef getTypesDef(ImmutableList<EnumTypeDefinition> enums, http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3cdd9f57/typesystem/src/main/scala/org/apache/atlas/typesystem/builders/TypesBuilder.scala ---------------------------------------------------------------------- diff --git a/typesystem/src/main/scala/org/apache/atlas/typesystem/builders/TypesBuilder.scala b/typesystem/src/main/scala/org/apache/atlas/typesystem/builders/TypesBuilder.scala index cd711d6..f18151a 100644 --- a/typesystem/src/main/scala/org/apache/atlas/typesystem/builders/TypesBuilder.scala +++ b/typesystem/src/main/scala/org/apache/atlas/typesystem/builders/TypesBuilder.scala @@ -22,10 +22,10 @@ import com.google.common.collect.ImmutableList import org.apache.atlas.typesystem.TypesDef import org.apache.atlas.typesystem.types._ import org.apache.atlas.typesystem.types.utils.TypesUtil - import scala.collection.mutable.ArrayBuffer import scala.language.{dynamics, implicitConversions, postfixOps} import scala.util.DynamicVariable +import com.google.common.collect.ImmutableSet object TypesBuilder { @@ -158,14 +158,14 @@ class TypesBuilder { val attrs = new ArrayBuffer[Attr]() context.withValue(context.value.copy(currentTypeAttrs = attrs)){f} context.value.classes += - TypesUtil.createClassTypeDef(name, ImmutableList.copyOf[String](superTypes.toArray), attrs.map(_.getDef):_*) + TypesUtil.createClassTypeDef(name, ImmutableSet.copyOf[String](superTypes.toArray), attrs.map(_.getDef):_*) } def _trait(name : String, superTypes : List[String] = List())(f : => Unit): Unit = { val attrs = new ArrayBuffer[Attr]() context.withValue(context.value.copy(currentTypeAttrs = attrs)){f} context.value.traits += - TypesUtil.createTraitTypeDef(name, ImmutableList.copyOf[String](superTypes.toArray), attrs.map(_.getDef):_*) + TypesUtil.createTraitTypeDef(name, ImmutableSet.copyOf[String](superTypes.toArray), attrs.map(_.getDef):_*) val v = context.value v.traits.size } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3cdd9f57/typesystem/src/main/scala/org/apache/atlas/typesystem/json/TypesSerialization.scala ---------------------------------------------------------------------- diff --git a/typesystem/src/main/scala/org/apache/atlas/typesystem/json/TypesSerialization.scala b/typesystem/src/main/scala/org/apache/atlas/typesystem/json/TypesSerialization.scala index 5d50c45..5618938 100755 --- a/typesystem/src/main/scala/org/apache/atlas/typesystem/json/TypesSerialization.scala +++ b/typesystem/src/main/scala/org/apache/atlas/typesystem/json/TypesSerialization.scala @@ -19,7 +19,6 @@ package org.apache.atlas.typesystem.json import java.text.SimpleDateFormat - import com.google.common.collect.ImmutableList import org.apache.atlas.AtlasException import org.apache.atlas.typesystem.TypesDef @@ -28,6 +27,7 @@ import org.apache.atlas.typesystem.types._ import org.json4s.JsonAST.JString import org.json4s._ import org.json4s.native.Serialization._ +import com.google.common.collect.ImmutableSet /** @@ -58,13 +58,13 @@ object TypesSerialization { def toJsonValue(typ: IDataType[_])(implicit formats: Formats): JValue = { typ.getTypeCategory match { case TypeCategory.CLASS => { - Extraction.decompose(convertClassTypeToHierarchicalTypeDefintion(typ.asInstanceOf[ClassType])) + Extraction.decompose(convertClassTypeToHierarchicalTypeDefinition(typ.asInstanceOf[ClassType])) } case TypeCategory.STRUCT => { Extraction.decompose(convertStructTypeToStructDef(typ.asInstanceOf[StructType])) } case TypeCategory.TRAIT => { - Extraction.decompose(convertTraitTypeToHierarchicalTypeDefintion(typ.asInstanceOf[TraitType])) + Extraction.decompose(convertTraitTypeToHierarchicalTypeDefinition(typ.asInstanceOf[TraitType])) } case TypeCategory.ENUM => { Extraction.decompose(convertEnumTypeToEnumTypeDef(typ.asInstanceOf[EnumType])) @@ -141,14 +141,14 @@ object TypesSerialization { new StructTypeDefinition(st.name, st.description, aDefs.toArray) } - private def convertTraitTypeToHierarchicalTypeDefintion(tt: TraitType): HierarchicalTypeDefinition[TraitType] = { + private def convertTraitTypeToHierarchicalTypeDefinition(tt: TraitType): HierarchicalTypeDefinition[TraitType] = { val aDefs: Iterable[AttributeDefinition] = tt.immediateAttrs.map(convertAttributeInfoToAttributeDef(_)) new HierarchicalTypeDefinition[TraitType](classOf[TraitType], tt.name, tt.description, tt.superTypes, aDefs.toArray) } - private def convertClassTypeToHierarchicalTypeDefintion(tt: ClassType): HierarchicalTypeDefinition[ClassType] = { + private def convertClassTypeToHierarchicalTypeDefinition(tt: ClassType): HierarchicalTypeDefinition[ClassType] = { val aDefs: Iterable[AttributeDefinition] = tt.immediateAttrs.map(convertAttributeInfoToAttributeDef(_)) @@ -173,8 +173,8 @@ object TypesSerialization { case typ: MapType => () case typ: EnumType => enumTypes = enumTypes :+ convertEnumTypeToEnumTypeDef(typ) case typ: StructType => structTypes = structTypes :+ convertStructTypeToStructDef(typ) - case typ: TraitType => traitTypes = traitTypes :+ convertTraitTypeToHierarchicalTypeDefintion(typ) - case typ: ClassType => classTypes = classTypes :+ convertClassTypeToHierarchicalTypeDefintion(typ) + case typ: TraitType => traitTypes = traitTypes :+ convertTraitTypeToHierarchicalTypeDefinition(typ) + case typ: ClassType => classTypes = classTypes :+ convertClassTypeToHierarchicalTypeDefinition(typ) } TypesDef(enumTypes, structTypes, traitTypes, classTypes) @@ -232,7 +232,7 @@ trait TypeHelpers { def createTraitTypeDef(name: String, description: Option[String], superTypes: Seq[String], attrDefs: AttributeDefinition*): HierarchicalTypeDefinition[TraitType] = { - val sts = ImmutableList.copyOf(superTypes.toArray) + val sts = ImmutableSet.copyOf(superTypes.toArray) return new HierarchicalTypeDefinition[TraitType](classOf[TraitType], name, description.getOrElse(null), sts, attrDefs.toArray) } @@ -244,7 +244,7 @@ trait TypeHelpers { def createClassTypeDef(name: String, description: Option[String], superTypes: Seq[String], attrDefs: AttributeDefinition*): HierarchicalTypeDefinition[ClassType] = { - val sts = ImmutableList.copyOf(superTypes.toArray) + val sts = ImmutableSet.copyOf(superTypes.toArray) return new HierarchicalTypeDefinition[ClassType](classOf[ClassType], name, description.getOrElse(null), sts, attrDefs.toArray) } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3cdd9f57/typesystem/src/test/java/org/apache/atlas/typesystem/json/SerializationJavaTest.java ---------------------------------------------------------------------- diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/json/SerializationJavaTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/json/SerializationJavaTest.java index a131146..eb1a15a 100755 --- a/typesystem/src/test/java/org/apache/atlas/typesystem/json/SerializationJavaTest.java +++ b/typesystem/src/test/java/org/apache/atlas/typesystem/json/SerializationJavaTest.java @@ -19,6 +19,8 @@ package org.apache.atlas.typesystem.json; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + import org.apache.atlas.AtlasException; import org.apache.atlas.typesystem.ITypedInstance; import org.apache.atlas.typesystem.ITypedReferenceableInstance; @@ -63,21 +65,21 @@ public class SerializationJavaTest extends BaseTest { TypeSystem ts = getTypeSystem(); - HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department", ImmutableList.<String>of(), + HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department", ImmutableSet.<String>of(), createRequiredAttrDef("name", DataTypes.STRING_TYPE), new AttributeDefinition("employees", String.format("array<%s>", "Person"), Multiplicity.COLLECTION, true, "department")); - HierarchicalTypeDefinition<ClassType> personTypeDef = createClassTypeDef("Person", ImmutableList.<String>of(), + HierarchicalTypeDefinition<ClassType> personTypeDef = createClassTypeDef("Person", ImmutableSet.<String>of(), createRequiredAttrDef("name", DataTypes.STRING_TYPE), new AttributeDefinition("department", "Department", Multiplicity.REQUIRED, false, "employees"), new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates")); HierarchicalTypeDefinition<ClassType> managerTypeDef = - createClassTypeDef("Manager", ImmutableList.<String>of("Person"), + createClassTypeDef("Manager", ImmutableSet.<String>of("Person"), new AttributeDefinition("subordinates", String.format("array<%s>", "Person"), Multiplicity.COLLECTION, false, "manager")); HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef = - createTraitTypeDef("SecurityClearance", ImmutableList.<String>of(), + createTraitTypeDef("SecurityClearance", ImmutableSet.<String>of(), createRequiredAttrDef("level", DataTypes.INT_TYPE)); ts.defineTypes(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), @@ -145,7 +147,7 @@ public class SerializationJavaTest extends BaseTest { TypeSystem ts = getTypeSystem(); HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef = - createTraitTypeDef("SecurityClearance2", ImmutableList.<String>of(), + createTraitTypeDef("SecurityClearance2", ImmutableSet.<String>of(), createRequiredAttrDef("level", DataTypes.INT_TYPE)); ts.defineTypes(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3cdd9f57/typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java ---------------------------------------------------------------------- diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java index 93dd099..e3352b0 100755 --- a/typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java +++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java @@ -19,8 +19,10 @@ package org.apache.atlas.typesystem.types; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; + import org.apache.atlas.AtlasException; import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Struct; @@ -118,22 +120,22 @@ public abstract class BaseTest { protected void defineDeptEmployeeTypes(TypeSystem ts) throws AtlasException { HierarchicalTypeDefinition<ClassType> deptTypeDef = TypesUtil - .createClassTypeDef("Department", ImmutableList.<String>of(), + .createClassTypeDef("Department", ImmutableSet.<String>of(), TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE), new AttributeDefinition("employees", String.format("array<%s>", "Person"), Multiplicity.COLLECTION, true, "department")); HierarchicalTypeDefinition<ClassType> personTypeDef = TypesUtil - .createClassTypeDef("Person", ImmutableList.<String>of(), + .createClassTypeDef("Person", ImmutableSet.<String>of(), TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE), new AttributeDefinition("department", "Department", Multiplicity.REQUIRED, false, "employees"), new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates")); HierarchicalTypeDefinition<ClassType> managerTypeDef = TypesUtil - .createClassTypeDef("Manager", ImmutableList.of("Person"), + .createClassTypeDef("Manager", ImmutableSet.of("Person"), new AttributeDefinition("subordinates", String.format("array<%s>", "Person"), Multiplicity.COLLECTION, false, "manager")); HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef = TypesUtil - .createTraitTypeDef("SecurityClearance", ImmutableList.<String>of(), + .createTraitTypeDef("SecurityClearance", ImmutableSet.<String>of(), TypesUtil.createRequiredAttrDef("level", DataTypes.INT_TYPE)); ts.defineTypes(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3cdd9f57/typesystem/src/test/java/org/apache/atlas/typesystem/types/ClassTest.java ---------------------------------------------------------------------- diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/ClassTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/ClassTest.java index a0f5843..daecdd7 100755 --- a/typesystem/src/test/java/org/apache/atlas/typesystem/types/ClassTest.java +++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/ClassTest.java @@ -29,6 +29,7 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; public class ClassTest extends HierarchicalTypeTest<ClassType> { @@ -78,7 +79,7 @@ public class ClassTest extends HierarchicalTypeTest<ClassType> { @Test public void testSerDeWithoutDescription() throws Exception { HierarchicalTypeDefinition<ClassType> clsType = TypesUtil - .createClassTypeDef("Random", ImmutableList.<String>of(), + .createClassTypeDef("Random", ImmutableSet.<String>of(), TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE)); TypesDef typesDef = getTypesDef(clsType); @@ -90,7 +91,7 @@ public class ClassTest extends HierarchicalTypeTest<ClassType> { @Test public void testSerDeWithDescription() throws Exception { HierarchicalTypeDefinition<ClassType> clsType = TypesUtil - .createClassTypeDef("Random", "Random-description", ImmutableList.<String>of(), + .createClassTypeDef("Random", "Random-description", ImmutableSet.<String>of(), TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE)); TypesDef typesDef = getTypesDef(clsType); String json = TypesSerialization.toJson(typesDef); @@ -103,7 +104,7 @@ public class ClassTest extends HierarchicalTypeTest<ClassType> { } @Override - protected HierarchicalTypeDefinition<ClassType> getTypeDefinition(String name, ImmutableList<String> superTypes, + protected HierarchicalTypeDefinition<ClassType> getTypeDefinition(String name, ImmutableSet<String> superTypes, AttributeDefinition... attributes) { return new HierarchicalTypeDefinition(ClassType.class, name, null, superTypes, attributes); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3cdd9f57/typesystem/src/test/java/org/apache/atlas/typesystem/types/EnumTest.java ---------------------------------------------------------------------- diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/EnumTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/EnumTest.java index ea7a798..adbeb19 100755 --- a/typesystem/src/test/java/org/apache/atlas/typesystem/types/EnumTest.java +++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/EnumTest.java @@ -19,8 +19,10 @@ package org.apache.atlas.typesystem.types; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; + import org.apache.atlas.AtlasException; import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.atlas.typesystem.ITypedReferenceableInstance; @@ -143,7 +145,7 @@ public class EnumTest extends BaseTest { protected ClassType defineClassTypeWithEnum(TypeSystem ts) throws AtlasException { return ts.defineClassType( - createClassTypeDef("t4", ImmutableList.<String>of(), createRequiredAttrDef("a", DataTypes.INT_TYPE), + createClassTypeDef("t4", ImmutableSet.<String>of(), createRequiredAttrDef("a", DataTypes.INT_TYPE), createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE), createOptionalAttrDef("c", DataTypes.BYTE_TYPE), createOptionalAttrDef("d", DataTypes.SHORT_TYPE),
