ATLAS-479 Add description for different types during create time (guptaneeru via shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/b6fb1f1a Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/b6fb1f1a Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/b6fb1f1a Branch: refs/heads/master Commit: b6fb1f1ac27f19e1b077f14d47c88548ae2df106 Parents: b9575f2 Author: Shwetha GS <[email protected]> Authored: Mon Mar 7 12:13:51 2016 +0530 Committer: Shwetha GS <[email protected]> Committed: Mon Mar 7 12:13:51 2016 +0530 ---------------------------------------------------------------------- .../falcon/model/FalconDataModelGenerator.java | 2 +- .../hive/model/HiveDataModelGenerator.java | 16 ++-- .../sqoop/model/SqoopDataModelGenerator.java | 4 +- release-log.txt | 1 + .../org/apache/atlas/repository/Constants.java | 1 + .../typestore/GraphBackedTypeStore.java | 43 ++++++--- .../apache/atlas/BaseHiveRepositoryTest.java | 2 +- .../test/java/org/apache/atlas/TestUtils.java | 35 ++++---- .../typestore/GraphBackedTypeStoreTest.java | 12 ++- .../service/DefaultMetadataServiceTest.java | 1 - .../apache/atlas/query/QueryTestsUtils.scala | 24 ++--- .../typesystem/types/AbstractDataType.java | 21 +++++ .../atlas/typesystem/types/ClassType.java | 12 ++- .../atlas/typesystem/types/DataTypes.java | 93 +++++--------------- .../apache/atlas/typesystem/types/EnumType.java | 13 ++- .../typesystem/types/EnumTypeDefinition.java | 6 ++ .../typesystem/types/HierarchicalType.java | 23 ++--- .../types/HierarchicalTypeDefinition.java | 11 +-- .../atlas/typesystem/types/IDataType.java | 2 + .../atlas/typesystem/types/StructType.java | 26 ++---- .../typesystem/types/StructTypeDefinition.java | 13 ++- .../atlas/typesystem/types/TraitType.java | 11 +-- .../atlas/typesystem/types/TypeSystem.java | 31 ++++--- .../atlas/typesystem/types/utils/TypesUtil.java | 20 ++++- .../typesystem/json/TypesSerialization.scala | 31 +++++-- .../atlas/typesystem/types/ClassTest.java | 31 ++++++- .../atlas/typesystem/types/TraitTest.java | 4 +- .../atlas/typesystem/types/TypeSystemTest.java | 46 +++++++++- .../json/TypesSerializationTest.scala | 57 ++++++++++-- .../org/apache/atlas/examples/QuickStart.java | 26 +++--- 30 files changed, 382 insertions(+), 236 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java index ac9dd85..a68db2f 100644 --- a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java +++ b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java @@ -115,7 +115,7 @@ public class FalconDataModelGenerator { Multiplicity.OPTIONAL, false, null),}; HierarchicalTypeDefinition<ClassType> definition = - new HierarchicalTypeDefinition<>(ClassType.class, FalconDataTypes.FALCON_PROCESS_ENTITY.getName(), + new HierarchicalTypeDefinition<>(ClassType.class, FalconDataTypes.FALCON_PROCESS_ENTITY.getName(), null, ImmutableList.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions); classTypeDefinitions.put(FalconDataTypes.FALCON_PROCESS_ENTITY.getName(), definition); LOG.debug("Created definition for {}", FalconDataTypes.FALCON_PROCESS_ENTITY.getName()); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java index 94ca7a4..a63c7cc 100755 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java @@ -201,7 +201,7 @@ public class HiveDataModelGenerator { Multiplicity.OPTIONAL, false, null),}; HierarchicalTypeDefinition<ClassType> definition = - new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_STORAGEDESC.getName(), + new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_STORAGEDESC.getName(), null, ImmutableList.of(AtlasClient.REFERENCEABLE_SUPER_TYPE), attributeDefinitions); classTypeDefinitions.put(HiveDataTypes.HIVE_STORAGEDESC.getName(), definition); LOG.debug("Created definition for " + HiveDataTypes.HIVE_STORAGEDESC.getName()); @@ -236,7 +236,7 @@ public class HiveDataModelGenerator { false, null),}; HierarchicalTypeDefinition<ClassType> definition = - new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_DB.getName(), + new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_DB.getName(), null, ImmutableList.of(AtlasClient.REFERENCEABLE_SUPER_TYPE), attributeDefinitions); classTypeDefinitions.put(HiveDataTypes.HIVE_DB.getName(), definition); LOG.debug("Created definition for " + HiveDataTypes.HIVE_DB.getName()); @@ -250,7 +250,7 @@ public class HiveDataModelGenerator { new AttributeDefinition("fields", String.format("array<%s>", HiveDataTypes.HIVE_COLUMN.getName()), Multiplicity.OPTIONAL, false, null),}; HierarchicalTypeDefinition<ClassType> definition = - new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TYPE.getName(), null, + new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TYPE.getName(), null, null, attributeDefinitions); classTypeDefinitions.put(HiveDataTypes.HIVE_TYPE.getName(), definition); @@ -263,7 +263,7 @@ public class HiveDataModelGenerator { new AttributeDefinition("type", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), new AttributeDefinition(COMMENT, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),}; HierarchicalTypeDefinition<ClassType> definition = - new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_COLUMN.getName(), + new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_COLUMN.getName(), null, ImmutableList.of(AtlasClient.REFERENCEABLE_SUPER_TYPE), attributeDefinitions); classTypeDefinitions.put(HiveDataTypes.HIVE_COLUMN.getName(), definition); LOG.debug("Created definition for " + HiveDataTypes.HIVE_COLUMN.getName()); @@ -284,7 +284,7 @@ public class HiveDataModelGenerator { Multiplicity.OPTIONAL, true, null), new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null),}; HierarchicalTypeDefinition<ClassType> definition = - new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_PARTITION.getName(), + new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_PARTITION.getName(), null, ImmutableList.of(AtlasClient.REFERENCEABLE_SUPER_TYPE), attributeDefinitions); classTypeDefinitions.put(HiveDataTypes.HIVE_PARTITION.getName(), definition); LOG.debug("Created definition for " + HiveDataTypes.HIVE_PARTITION.getName()); @@ -318,7 +318,7 @@ public class HiveDataModelGenerator { new AttributeDefinition("temporary", DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.OPTIONAL, false, null),}; HierarchicalTypeDefinition<ClassType> definition = - new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TABLE.getName(), + new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TABLE.getName(), null, ImmutableList.of("DataSet"), attributeDefinitions); classTypeDefinitions.put(HiveDataTypes.HIVE_TABLE.getName(), definition); LOG.debug("Created definition for " + HiveDataTypes.HIVE_TABLE.getName()); @@ -333,7 +333,7 @@ public class HiveDataModelGenerator { new AttributeDefinition("ownerName", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),}; HierarchicalTypeDefinition<ClassType> definition = - new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_ROLE.getName(), null, + new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_ROLE.getName(), null, null, attributeDefinitions); classTypeDefinitions.put(HiveDataTypes.HIVE_ROLE.getName(), definition); @@ -357,7 +357,7 @@ public class HiveDataModelGenerator { null),}; HierarchicalTypeDefinition<ClassType> definition = - new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_PROCESS.getName(), + new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_PROCESS.getName(), null, ImmutableList.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions); classTypeDefinitions.put(HiveDataTypes.HIVE_PROCESS.getName(), definition); LOG.debug("Created definition for " + HiveDataTypes.HIVE_PROCESS.getName()); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java ---------------------------------------------------------------------- diff --git a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java index 342c07f..989469f 100644 --- a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java +++ b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java @@ -129,7 +129,7 @@ public class SqoopDataModelGenerator { DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),}; HierarchicalTypeDefinition<ClassType> definition = - new HierarchicalTypeDefinition<>(ClassType.class, SqoopDataTypes.SQOOP_DBDATASTORE.getName(), + new HierarchicalTypeDefinition<>(ClassType.class, SqoopDataTypes.SQOOP_DBDATASTORE.getName(), null, ImmutableList.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions); classTypeDefinitions.put(SqoopDataTypes.SQOOP_DBDATASTORE.getName(), definition); LOG.debug("Created definition for " + SqoopDataTypes.SQOOP_DBDATASTORE.getName()); @@ -148,7 +148,7 @@ public class SqoopDataModelGenerator { }; HierarchicalTypeDefinition<ClassType> definition = - new HierarchicalTypeDefinition<>(ClassType.class, SqoopDataTypes.SQOOP_PROCESS.getName(), + new HierarchicalTypeDefinition<>(ClassType.class, SqoopDataTypes.SQOOP_PROCESS.getName(), null, ImmutableList.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions); classTypeDefinitions.put(SqoopDataTypes.SQOOP_PROCESS.getName(), definition); LOG.debug("Created definition for " + SqoopDataTypes.SQOOP_PROCESS.getName()); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 92a91e9..a43abfc 100644 --- a/release-log.txt +++ b/release-log.txt @@ -9,6 +9,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ALL CHANGES: +ATLAS-479 Add description for different types during create time (guptaneeru via shwethags) ATLAS-508 Apache nightly build failure - UnsupportedOperationException: Not a single key: __traitNames (shwethags) ATLAS-422 JavaDoc NotificationConsumer and NotificationInterface.(tbeerbower via sumasai) ATLAS-536 Falcon hook loads incorrect configuration when -Datlas.conf is not given when falcon server startup (ayubkhan via shwethags) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/repository/src/main/java/org/apache/atlas/repository/Constants.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/Constants.java b/repository/src/main/java/org/apache/atlas/repository/Constants.java index 1b4cb36..aa3326e 100755 --- a/repository/src/main/java/org/apache/atlas/repository/Constants.java +++ b/repository/src/main/java/org/apache/atlas/repository/Constants.java @@ -49,6 +49,7 @@ public final class Constants { public static final String TYPE_CATEGORY_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.category"; public static final String VERTEX_TYPE_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type"; public static final String TYPENAME_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.name"; + public static final String TYPEDESCRIPTION_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.description"; /** * Trait names property key and index name. http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java index 9789be7..bf10c8d 100755 --- a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java @@ -79,7 +79,7 @@ public class GraphBackedTypeStore implements ITypeStore { public void store(TypeSystem typeSystem, ImmutableList<String> typeNames) throws AtlasException { for (String typeName : typeNames) { IDataType dataType = typeSystem.getDataType(IDataType.class, typeName); - LOG.debug("Processing {}.{} in type store", dataType.getTypeCategory(), dataType.getName()); + LOG.debug("Processing {}.{}.{} in type store", dataType.getTypeCategory(), dataType.getName(), dataType.getDescription()); switch (dataType.getTypeCategory()) { case ENUM: storeInGraph((EnumType) dataType); @@ -87,14 +87,14 @@ public class GraphBackedTypeStore implements ITypeStore { case STRUCT: StructType structType = (StructType) dataType; - storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(), + storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(), dataType.getDescription(), ImmutableList.copyOf(structType.infoToNameMap.keySet()), ImmutableList.<String>of()); break; case TRAIT: case CLASS: HierarchicalType type = (HierarchicalType) dataType; - storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(), type.immediateAttrs, + storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(), type.getDescription(), type.immediateAttrs, type.superTypes); break; @@ -110,7 +110,7 @@ public class GraphBackedTypeStore implements ITypeStore { } private void storeInGraph(EnumType dataType) { - Vertex vertex = createVertex(dataType.getTypeCategory(), dataType.getName()); + Vertex vertex = createVertex(dataType.getTypeCategory(), dataType.getName(), dataType.getDescription()); List<String> values = new ArrayList<>(dataType.values().size()); for (EnumValue enumValue : dataType.values()) { String key = getPropertyKey(dataType.getName(), enumValue.value); @@ -133,8 +133,13 @@ public class GraphBackedTypeStore implements ITypeStore { } private void storeInGraph(TypeSystem typeSystem, DataTypes.TypeCategory category, String typeName, + ImmutableList<AttributeInfo> attributes, ImmutableList<String> superTypes) throws AtlasException { + storeInGraph(typeSystem, category, typeName, null, attributes, superTypes); + } + + private void storeInGraph(TypeSystem typeSystem, DataTypes.TypeCategory category, String typeName, String typeDescription, ImmutableList<AttributeInfo> attributes, ImmutableList<String> superTypes) throws AtlasException { - Vertex vertex = createVertex(category, typeName); + Vertex vertex = createVertex(category, typeName, typeDescription); List<String> attrNames = new ArrayList<>(); if (attributes != null) { for (AttributeInfo attribute : attributes) { @@ -154,7 +159,7 @@ public class GraphBackedTypeStore implements ITypeStore { if (superTypes != null) { for (String superTypeName : superTypes) { HierarchicalType superType = typeSystem.getDataType(HierarchicalType.class, superTypeName); - Vertex superVertex = createVertex(superType.getTypeCategory(), superTypeName); + Vertex superVertex = createVertex(superType.getTypeCategory(), superTypeName, superType.getDescription()); addEdge(vertex, superVertex, SUPERTYPE_EDGE_LABEL); } } @@ -199,7 +204,7 @@ public class GraphBackedTypeStore implements ITypeStore { for (IDataType attrType : attrDataTypes) { if (!coreTypes.contains(attrType.getName())) { - Vertex attrVertex = createVertex(attrType.getTypeCategory(), attrType.getName()); + Vertex attrVertex = createVertex(attrType.getTypeCategory(), attrType.getName(), attrType.getDescription()); String label = getEdgeLabel(vertexTypeName, attribute.name); addEdge(vertex, attrVertex, label); } @@ -227,7 +232,8 @@ public class GraphBackedTypeStore implements ITypeStore { Vertex vertex = (Vertex) vertices.next(); DataTypes.TypeCategory typeCategory = vertex.getProperty(Constants.TYPE_CATEGORY_PROPERTY_KEY); String typeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY); - LOG.info("Restoring type {}.{}", typeCategory, typeName); + String typeDescription = vertex.getProperty(Constants.TYPEDESCRIPTION_PROPERTY_KEY); + LOG.info("Restoring type {}.{}.{}", typeCategory, typeName, typeDescription); switch (typeCategory) { case ENUM: enums.add(getEnumType(vertex)); @@ -235,19 +241,19 @@ public class GraphBackedTypeStore implements ITypeStore { case STRUCT: AttributeDefinition[] attributes = getAttributes(vertex, typeName); - structs.add(new StructTypeDefinition(typeName, attributes)); + structs.add(new StructTypeDefinition(typeName, typeDescription, attributes)); break; case CLASS: ImmutableList<String> superTypes = getSuperTypes(vertex); attributes = getAttributes(vertex, typeName); - classTypes.add(new HierarchicalTypeDefinition(ClassType.class, typeName, superTypes, attributes)); + classTypes.add(new HierarchicalTypeDefinition(ClassType.class, typeName, typeDescription, superTypes, attributes)); break; case TRAIT: superTypes = getSuperTypes(vertex); attributes = getAttributes(vertex, typeName); - traits.add(new HierarchicalTypeDefinition(TraitType.class, typeName, superTypes, attributes)); + traits.add(new HierarchicalTypeDefinition(TraitType.class, typeName, typeDescription, superTypes, attributes)); break; default: @@ -259,13 +265,14 @@ public class GraphBackedTypeStore implements ITypeStore { private EnumTypeDefinition getEnumType(Vertex vertex) { String typeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY); + String typeDescription = vertex.getProperty(Constants.TYPEDESCRIPTION_PROPERTY_KEY); List<EnumValue> enumValues = new ArrayList<>(); List<String> values = vertex.getProperty(getPropertyKey(typeName)); for (String value : values) { String valueProperty = getPropertyKey(typeName, value); enumValues.add(new EnumValue(value, vertex.<Integer>getProperty(valueProperty))); } - return new EnumTypeDefinition(typeName, enumValues.toArray(new EnumValue[enumValues.size()])); + return new EnumTypeDefinition(typeName, typeDescription, enumValues.toArray(new EnumValue[enumValues.size()])); } private ImmutableList<String> getSuperTypes(Vertex vertex) { @@ -316,15 +323,23 @@ public class GraphBackedTypeStore implements ITypeStore { return vertex; } - private Vertex createVertex(DataTypes.TypeCategory category, String typeName) { + private Vertex createVertex(DataTypes.TypeCategory category, String typeName, String typeDescription) { Vertex vertex = findVertex(category, typeName); if (vertex == null) { LOG.debug("Adding vertex {}{}", PROPERTY_PREFIX, typeName); vertex = titanGraph.addVertex(null); - addProperty(vertex, Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE); //Mark as type vertex + addProperty(vertex, Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE); // Mark as type vertex addProperty(vertex, Constants.TYPE_CATEGORY_PROPERTY_KEY, category); addProperty(vertex, Constants.TYPENAME_PROPERTY_KEY, typeName); } + if (typeDescription != null) { + String oldDescription = getPropertyKey(Constants.TYPEDESCRIPTION_PROPERTY_KEY); + if (!typeDescription.equals(oldDescription)) { + addProperty(vertex, Constants.TYPEDESCRIPTION_PROPERTY_KEY, typeDescription); + } + } else { + LOG.debug(" type description is null "); + } return vertex; } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java b/repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java index 70849ca..b386cf2 100644 --- a/repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java +++ b/repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java @@ -151,7 +151,7 @@ public class BaseHiveRepositoryTest { new AttributeDefinition("table", HIVE_TABLE_TYPE, Multiplicity.REQUIRED, false, null), }; HierarchicalTypeDefinition<ClassType> partClsDef = - new HierarchicalTypeDefinition<>(ClassType.class, PARTITION_TYPE, null, + new HierarchicalTypeDefinition<>(ClassType.class, PARTITION_TYPE, null, null, attributeDefinitions); HierarchicalTypeDefinition<TraitType> dimTraitDef = TypesUtil.createTraitTypeDef("Dimension", null); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/repository/src/test/java/org/apache/atlas/TestUtils.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/TestUtils.java b/repository/src/test/java/org/apache/atlas/TestUtils.java index 653ec01..d54dfab 100755 --- a/repository/src/test/java/org/apache/atlas/TestUtils.java +++ b/repository/src/test/java/org/apache/atlas/TestUtils.java @@ -23,7 +23,6 @@ import com.thinkaurelius.titan.core.TitanGraph; import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter; - import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.Referenceable; @@ -97,19 +96,20 @@ public final class TestUtils { */ public static void defineDeptEmployeeTypes(TypeSystem ts) throws AtlasException { + String _description = "_description"; EnumTypeDefinition orgLevelEnum = - new EnumTypeDefinition("OrgLevel", new EnumValue("L1", 1), new EnumValue("L2", 2)); + new EnumTypeDefinition("OrgLevel", "OrgLevel"+_description, new EnumValue("L1", 1), new EnumValue("L2", 2)); StructTypeDefinition addressDetails = - createStructTypeDef("Address", createRequiredAttrDef("street", DataTypes.STRING_TYPE), + createStructTypeDef("Address", "Address"+_description, createRequiredAttrDef("street", DataTypes.STRING_TYPE), createRequiredAttrDef("city", DataTypes.STRING_TYPE)); - HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department", ImmutableList.<String>of(), + HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department", "Department"+_description, ImmutableList.<String>of(), createRequiredAttrDef("name", DataTypes.STRING_TYPE), new AttributeDefinition("employees", String.format("array<%s>", "Person"), Multiplicity.COLLECTION, true, "department")); - HierarchicalTypeDefinition<ClassType> personTypeDef = createClassTypeDef("Person", ImmutableList.<String>of(), + HierarchicalTypeDefinition<ClassType> personTypeDef = createClassTypeDef("Person", "Person"+_description, ImmutableList.<String>of(), createRequiredAttrDef("name", DataTypes.STRING_TYPE), createOptionalAttrDef("orgLevel", "OrgLevel"), createOptionalAttrDef("address", "Address"), @@ -117,12 +117,12 @@ public final class TestUtils { new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates"), new AttributeDefinition("mentor", "Person", Multiplicity.OPTIONAL, false, null)); - HierarchicalTypeDefinition<ClassType> managerTypeDef = createClassTypeDef("Manager", ImmutableList.of("Person"), + HierarchicalTypeDefinition<ClassType> managerTypeDef = createClassTypeDef("Manager", "Manager"+_description, ImmutableList.of("Person"), new AttributeDefinition("subordinates", String.format("array<%s>", "Person"), Multiplicity.COLLECTION, false, "manager")); HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef = - createTraitTypeDef("SecurityClearance", ImmutableList.<String>of(), + createTraitTypeDef("SecurityClearance", "SecurityClearance"+_description, ImmutableList.<String>of(), createRequiredAttrDef("level", DataTypes.INT_TYPE)); ts.defineTypes(ImmutableList.of(orgLevelEnum), ImmutableList.of(addressDetails), @@ -199,6 +199,7 @@ public final class TestUtils { public static final String SERDE_TYPE = "serdeType"; public static TypesDef defineHiveTypes() { + String _description = "_description"; HierarchicalTypeDefinition<ClassType> superTypeDefinition = createClassTypeDef(SUPER_TYPE_NAME, ImmutableList.<String>of(), createOptionalAttrDef("namespace", DataTypes.STRING_TYPE), @@ -206,27 +207,27 @@ public final class TestUtils { createOptionalAttrDef("colo", DataTypes.STRING_TYPE)); HierarchicalTypeDefinition<ClassType> databaseTypeDefinition = - createClassTypeDef(DATABASE_TYPE, ImmutableList.of(SUPER_TYPE_NAME), + createClassTypeDef(DATABASE_TYPE, DATABASE_TYPE + _description,ImmutableList.of(SUPER_TYPE_NAME), TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE), createOptionalAttrDef("created", DataTypes.DATE_TYPE), createRequiredAttrDef("description", DataTypes.STRING_TYPE)); - StructTypeDefinition structTypeDefinition = new StructTypeDefinition("serdeType", + StructTypeDefinition structTypeDefinition = new StructTypeDefinition("serdeType", "serdeType" + _description, new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE), createRequiredAttrDef("serde", DataTypes.STRING_TYPE), createOptionalAttrDef("description", DataTypes.STRING_TYPE)}); EnumValue values[] = {new EnumValue("MANAGED", 1), new EnumValue("EXTERNAL", 2),}; - EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", values); + EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", "tableType" + _description, values); HierarchicalTypeDefinition<ClassType> columnsDefinition = createClassTypeDef(COLUMN_TYPE, ImmutableList.<String>of(), createRequiredAttrDef("name", DataTypes.STRING_TYPE), createRequiredAttrDef("type", DataTypes.STRING_TYPE)); - StructTypeDefinition partitionDefinition = new StructTypeDefinition("partition_struct_type", + StructTypeDefinition partitionDefinition = new StructTypeDefinition("partition_struct_type", "partition_struct_type" + _description, new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE),}); AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ @@ -245,7 +246,7 @@ public final class TestUtils { }; HierarchicalTypeDefinition<ClassType> storageDescClsDef = - new HierarchicalTypeDefinition<>(ClassType.class, STORAGE_DESC_TYPE, + new HierarchicalTypeDefinition<>(ClassType.class, STORAGE_DESC_TYPE, STORAGE_DESC_TYPE + _description, ImmutableList.of(SUPER_TYPE_NAME), attributeDefinitions); AttributeDefinition[] partClsAttributes = new AttributeDefinition[]{ @@ -263,11 +264,11 @@ public final class TestUtils { new AttributeDefinition("parameters", new DataTypes.MapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE).getName(), Multiplicity.OPTIONAL, false, null),}; HierarchicalTypeDefinition<ClassType> partClsDef = - new HierarchicalTypeDefinition<>(ClassType.class, "partition_class_type", + new HierarchicalTypeDefinition<>(ClassType.class, "partition_class_type", "partition_class_type" + _description, ImmutableList.of(SUPER_TYPE_NAME), partClsAttributes); HierarchicalTypeDefinition<ClassType> tableTypeDefinition = - createClassTypeDef(TABLE_TYPE, ImmutableList.of(SUPER_TYPE_NAME), + createClassTypeDef(TABLE_TYPE, TABLE_TYPE + _description, ImmutableList.of(SUPER_TYPE_NAME), TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE), createRequiredAttrDef("description", DataTypes.STRING_TYPE), createRequiredAttrDef("type", DataTypes.STRING_TYPE), @@ -307,14 +308,14 @@ public final class TestUtils { new AttributeDefinition("databaseComposite", DATABASE_TYPE, Multiplicity.OPTIONAL, true, null)); HierarchicalTypeDefinition<TraitType> piiTypeDefinition = - createTraitTypeDef(PII, ImmutableList.<String>of()); + createTraitTypeDef(PII, PII + _description, ImmutableList.<String>of()); HierarchicalTypeDefinition<TraitType> classificationTypeDefinition = - createTraitTypeDef(CLASSIFICATION, ImmutableList.<String>of(), + createTraitTypeDef(CLASSIFICATION, CLASSIFICATION + _description, ImmutableList.<String>of(), createRequiredAttrDef("tag", DataTypes.STRING_TYPE)); HierarchicalTypeDefinition<TraitType> fetlClassificationTypeDefinition = - createTraitTypeDef("fetl" + CLASSIFICATION, ImmutableList.of(CLASSIFICATION), + createTraitTypeDef("fetl" + CLASSIFICATION, "fetl" + CLASSIFICATION + _description, ImmutableList.of(CLASSIFICATION), createRequiredAttrDef("tag", DataTypes.STRING_TYPE)); return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java b/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java index 8402eed..42183fd 100755 --- a/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java @@ -24,6 +24,7 @@ import com.thinkaurelius.titan.core.util.TitanCleanup; import com.tinkerpop.blueprints.Direction; import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Vertex; + import org.apache.atlas.AtlasException; import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.TestUtils; @@ -50,6 +51,7 @@ import org.testng.annotations.Guice; import org.testng.annotations.Test; import javax.inject.Inject; + import java.util.List; import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef; @@ -103,6 +105,7 @@ public class GraphBackedTypeStoreTest { @Test(dependsOnMethods = "testStore") public void testRestore() throws Exception { + String description = "_description"; TypesDef types = typeStore.restore(); //validate enum @@ -110,6 +113,7 @@ public class GraphBackedTypeStoreTest { Assert.assertEquals(1, enumTypes.size()); EnumTypeDefinition orgLevel = enumTypes.get(0); Assert.assertEquals(orgLevel.name, "OrgLevel"); + Assert.assertEquals(orgLevel.description, "OrgLevel"+description); Assert.assertEquals(orgLevel.enumValues.length, 2); EnumValue enumValue = orgLevel.enumValues[0]; Assert.assertEquals(enumValue.value, "L1"); @@ -126,6 +130,7 @@ public class GraphBackedTypeStoreTest { ClassType expectedType = ts.getDataType(ClassType.class, classType.typeName); Assert.assertEquals(expectedType.immediateAttrs.size(), classType.attributeDefinitions.length); Assert.assertEquals(expectedType.superTypes.size(), classType.superTypes.size()); + Assert.assertEquals(classType.typeDescription, classType.typeName+description); clsTypeFound = true; } } @@ -136,6 +141,7 @@ public class GraphBackedTypeStoreTest { Assert.assertEquals(1, traitTypes.size()); HierarchicalTypeDefinition<TraitType> trait = traitTypes.get(0); Assert.assertEquals("SecurityClearance", trait.typeName); + Assert.assertEquals(trait.typeName+description, trait.typeDescription); Assert.assertEquals(1, trait.attributeDefinitions.length); AttributeDefinition attribute = trait.attributeDefinitions[0]; Assert.assertEquals("level", attribute.name); @@ -149,7 +155,8 @@ public class GraphBackedTypeStoreTest { @Test(dependsOnMethods = "testStore") public void testTypeUpdate() throws Exception { //Add enum value - EnumTypeDefinition orgLevelEnum = new EnumTypeDefinition("OrgLevel", new EnumValue("L1", 1), + String _description = "_description_updated"; + EnumTypeDefinition orgLevelEnum = new EnumTypeDefinition("OrgLevel", "OrgLevel"+_description, new EnumValue("L1", 1), new EnumValue("L2", 2), new EnumValue("L3", 3)); //Add attribute @@ -162,7 +169,7 @@ public class GraphBackedTypeStoreTest { HierarchicalTypeDefinition<ClassType> superTypeDef = createClassTypeDef("Division", ImmutableList.<String>of(), createOptionalAttrDef("dname", DataTypes.STRING_TYPE)); - HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department", + HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department", "Department"+_description, ImmutableList.of(superTypeDef.typeName), createRequiredAttrDef("name", DataTypes.STRING_TYPE), new AttributeDefinition("employees", String.format("array<%s>", "Person"), Multiplicity.COLLECTION, true, "department")); @@ -182,6 +189,7 @@ public class GraphBackedTypeStoreTest { //Assert new enum value EnumType orgLevel = ts.getDataType(EnumType.class, orgLevelEnum.name); Assert.assertEquals(orgLevel.name, orgLevelEnum.name); + Assert.assertEquals(orgLevel.description, orgLevelEnum.description); Assert.assertEquals(orgLevel.values().size(), orgLevelEnum.enumValues.length); Assert.assertEquals(orgLevel.fromValue("L3").ordinal, 3); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java index d00f12e..019e21b 100644 --- a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java @@ -165,7 +165,6 @@ public class DefaultMetadataServiceTest { entity.set("type", "VARCHAR(32)"); return entity; } - @Test(expectedExceptions = TypeNotFoundException.class) public void testCreateEntityWithUnknownDatatype() throws Exception { Referenceable entity = new Referenceable("Unknown datatype"); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala ---------------------------------------------------------------------- diff --git a/repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala b/repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala index 87494d8..1a0fa20 100755 --- a/repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala +++ b/repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala @@ -78,7 +78,7 @@ object QueryTestsUtils extends GraphUtils { new AttributeDefinition(name, dT.getName, m, isComposite, reverseAttributeName) } - def dbClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "DB", null, + def dbClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "DB", null, null, Array( attrDef("name", DataTypes.STRING_TYPE), attrDef("owner", DataTypes.STRING_TYPE), @@ -92,21 +92,21 @@ object QueryTestsUtils extends GraphUtils { attrDef("order", DataTypes.INT_TYPE) )) - def storageDescClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "StorageDescriptor", null, + def storageDescClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "StorageDescriptor", null, null, Array( attrDef("inputFormat", DataTypes.STRING_TYPE), attrDef("outputFormat", DataTypes.STRING_TYPE), new AttributeDefinition("sortCols", DataTypes.arrayTypeName("HiveOrder"), Multiplicity.REQUIRED, false, null) )) - def columnClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "Column", null, + def columnClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "Column", null, null, Array( attrDef("name", DataTypes.STRING_TYPE), attrDef("dataType", DataTypes.STRING_TYPE), new AttributeDefinition("sd", "StorageDescriptor", Multiplicity.REQUIRED, false, null) )) - def tblClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "Table", null, + def tblClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "Table", null, null, Array( attrDef("name", DataTypes.STRING_TYPE), new AttributeDefinition("db", "DB", Multiplicity.REQUIRED, false, null), @@ -114,34 +114,34 @@ object QueryTestsUtils extends GraphUtils { attrDef("created", DataTypes.DATE_TYPE) )) - def partitionClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "Partition", null, + def partitionClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "Partition", null, null, Array( new AttributeDefinition("values", DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName), Multiplicity.REQUIRED, false, null), new AttributeDefinition("table", "Table", Multiplicity.REQUIRED, false, null) )) - def loadProcessClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "LoadProcess", null, + def loadProcessClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "LoadProcess", null, null, Array( attrDef("name", DataTypes.STRING_TYPE), new AttributeDefinition("inputTables", DataTypes.arrayTypeName("Table"), Multiplicity.COLLECTION, false, null), new AttributeDefinition("outputTable", "Table", Multiplicity.REQUIRED, false, null) )) - def viewClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "View", null, + def viewClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "View", null, null, Array( attrDef("name", DataTypes.STRING_TYPE), new AttributeDefinition("inputTables", DataTypes.arrayTypeName("Table"), Multiplicity.COLLECTION, false, null) )) - def dimTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "Dimension", null, + def dimTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "Dimension", null, null, Array[AttributeDefinition]()) - def piiTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "PII", null, + def piiTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "PII", null, null, Array[AttributeDefinition]()) - def metricTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "Metric", null, + def metricTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "Metric", null, null, Array[AttributeDefinition]()) - def etlTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "ETL", null, + def etlTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "ETL", null, null, Array[AttributeDefinition]()) - def jdbcTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "JdbcAccess", null, + def jdbcTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "JdbcAccess", null, null, Array[AttributeDefinition]()) TypeSystem.getInstance().defineTypes(ImmutableList.of[EnumTypeDefinition], http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AbstractDataType.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AbstractDataType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AbstractDataType.java index ed0400a..92be3c7 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AbstractDataType.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AbstractDataType.java @@ -19,12 +19,23 @@ package org.apache.atlas.typesystem.types; import com.google.common.collect.ImmutableSortedMap; + import org.apache.atlas.AtlasException; import java.util.Map; abstract class AbstractDataType<T> implements IDataType<T> { + public final String name; + public final String description; + + public AbstractDataType(String name, String description) { + + super(); + this.name = name; + this.description = description; + } + protected T convertNull(Multiplicity m) throws AtlasException { if (!m.nullAllowed()) { throw new ValueConversionException.NullConversionException(m); @@ -52,5 +63,15 @@ abstract class AbstractDataType<T> implements IDataType<T> { throw new TypeUpdateException(newType); } } + + @Override + public String getName() { + return name; + } + + @Override + public String getDescription() { + return description; + } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java index adf5f1c..d95e80a 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java @@ -21,6 +21,7 @@ package org.apache.atlas.typesystem.types; import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; + import org.apache.atlas.AtlasException; import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.atlas.typesystem.IStruct; @@ -47,17 +48,14 @@ public class ClassType extends HierarchicalType<ClassType, IReferenceableInstanc public final Map<AttributeInfo, List<String>> infoToNameMap; - /** - * Used when creating a ClassType, to support recursive Structs. - */ - ClassType(TypeSystem typeSystem, String name, ImmutableList<String> superTypes, int numFields) { - super(typeSystem, ClassType.class, name, superTypes, numFields); + ClassType(TypeSystem typeSystem, String name, String description, ImmutableList<String> superTypes, int numFields) { + super(typeSystem, ClassType.class, name, description, superTypes, numFields); infoToNameMap = null; } - ClassType(TypeSystem typeSystem, String name, ImmutableList<String> superTypes, AttributeInfo... fields) + ClassType(TypeSystem typeSystem, String name, String description, ImmutableList<String> superTypes, AttributeInfo... fields) throws AtlasException { - super(typeSystem, ClassType.class, name, superTypes, fields); + super(typeSystem, ClassType.class, name, description, superTypes, fields); infoToNameMap = TypeUtils.buildAttrInfoToNameMap(fieldMapping); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java index afd1682..55ec91f 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; + import org.apache.atlas.AtlasException; import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.atlas.typesystem.persistence.Id; @@ -57,6 +58,7 @@ public class DataTypes { static String MAP_TYPE_SUFFIX = ">"; public static String arrayTypeName(String elemTypeName) { + assert elemTypeName != null; return String.format("%s%s%s", ARRAY_TYPE_PREFIX, elemTypeName, ARRAY_TYPE_SUFFIX); } @@ -69,6 +71,8 @@ public class DataTypes { } public static String mapTypeName(IDataType keyType, IDataType valueType) { + assert keyType != null; + assert valueType != null; return mapTypeName(keyType.getName(), valueType.getName()); } @@ -83,6 +87,10 @@ public class DataTypes { } public static abstract class PrimitiveType<T> extends AbstractDataType<T> { + public PrimitiveType(String name, String description) { + super(name, description); + } + @Override public TypeCategory getTypeCategory() { return TypeCategory.PRIMITIVE; @@ -113,11 +121,7 @@ public class DataTypes { private static final String name = "boolean".intern(); private BooleanType() { - } - - @Override - public String getName() { - return name; + super(name, null); } @Override @@ -146,11 +150,7 @@ public class DataTypes { private static final String name = "byte".intern(); private ByteType() { - } - - @Override - public String getName() { - return name; + super(name, null); } @Override @@ -186,11 +186,7 @@ public class DataTypes { private static final String name = "short".intern(); private ShortType() { - } - - @Override - public String getName() { - return name; + super(name, null); } @Override @@ -219,11 +215,7 @@ public class DataTypes { private static final String name = "int".intern(); private IntType() { - } - - @Override - public String getName() { - return name; + super(name, null); } @Override @@ -252,11 +244,7 @@ public class DataTypes { private static final String name = "long".intern(); private LongType() { - } - - @Override - public String getName() { - return name; + super(name, null); } @Override @@ -285,11 +273,7 @@ public class DataTypes { private static final String name = "float".intern(); private FloatType() { - } - - @Override - public String getName() { - return name; + super(name, null); } @Override @@ -318,11 +302,7 @@ public class DataTypes { private static final String name = "double".intern(); private DoubleType() { - } - - @Override - public String getName() { - return name; + super(name, null); } @Override @@ -351,11 +331,7 @@ public class DataTypes { private static final String name = "biginteger".intern(); private BigIntegerType() { - } - - @Override - public String getName() { - return name; + super(name, null); } @Override @@ -390,11 +366,7 @@ public class DataTypes { private static final String name = "bigdecimal".intern(); private BigDecimalType() { - } - - @Override - public String getName() { - return name; + super(name, null); } @Override @@ -429,11 +401,7 @@ public class DataTypes { private static final String name = "date".intern(); private DateType() { - } - - @Override - public String getName() { - return name; + super(name, null); } @Override @@ -472,11 +440,7 @@ public class DataTypes { private static final String name = "string".intern(); private StringType() { - } - - @Override - public String getName() { - return name; + super(name, null); } @Override @@ -493,13 +457,11 @@ public class DataTypes { } public static class ArrayType extends AbstractDataType<ImmutableCollection<?>> { - private final String nm; private IDataType elemType; public ArrayType(IDataType elemType) { - assert elemType != null; + super(arrayTypeName(elemType), null); this.elemType = elemType; - this.nm = arrayTypeName(elemType); } public IDataType getElemType() { @@ -511,11 +473,6 @@ public class DataTypes { } @Override - public String getName() { - return nm; - } - - @Override public ImmutableCollection<?> convert(Object val, Multiplicity m) throws AtlasException { if (val != null) { Iterator it = null; @@ -589,16 +546,13 @@ public class DataTypes { public static class MapType extends AbstractDataType<ImmutableMap<?, ?>> { - private final String nm; private IDataType keyType; private IDataType valueType; public MapType(IDataType keyType, IDataType valueType) { - assert keyType != null; - assert valueType != null; + super(mapTypeName(keyType, valueType), null); this.keyType = keyType; this.valueType = valueType; - this.nm = mapTypeName(keyType, valueType); } public IDataType getKeyType() { @@ -618,11 +572,6 @@ public class DataTypes { } @Override - public String getName() { - return nm; - } - - @Override public ImmutableMap<?, ?> convert(Object val, Multiplicity m) throws AtlasException { if (val != null) { Iterator<Map.Entry> it = null; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java index b751307..bdd0a13 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java @@ -29,13 +29,16 @@ import java.security.MessageDigest; public class EnumType extends AbstractDataType<EnumValue> { public final TypeSystem typeSystem; - public final String name; public final ImmutableMap<String, EnumValue> valueMap; public final ImmutableMap<Integer, EnumValue> ordinalMap; protected EnumType(TypeSystem typeSystem, String name, EnumValue... values) { + this(typeSystem, name, null, values); + } + + protected EnumType(TypeSystem typeSystem, String name, String description, EnumValue... values) { + super(name, description); this.typeSystem = typeSystem; - this.name = name; ImmutableMap.Builder<String, EnumValue> b1 = new ImmutableMap.Builder(); ImmutableMap.Builder<Integer, EnumValue> b2 = new ImmutableMap.Builder(); for (EnumValue v : values) { @@ -46,12 +49,6 @@ public class EnumType extends AbstractDataType<EnumValue> { ordinalMap = b2.build(); } - - @Override - public String getName() { - return name; - } - @Override public EnumValue convert(Object val, Multiplicity m) throws AtlasException { if (val != null) { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumTypeDefinition.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumTypeDefinition.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumTypeDefinition.java index aca1a41..6340615 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumTypeDefinition.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumTypeDefinition.java @@ -25,10 +25,16 @@ import java.util.Arrays; public final class EnumTypeDefinition { public final String name; + public final String description; public final EnumValue[] enumValues; public EnumTypeDefinition(String name, EnumValue... enumValues) { + this(name, null, enumValues); + } + + public EnumTypeDefinition(String name, String description, EnumValue... enumValues) { this.name = ParamChecker.notEmpty(name, "Enum type name"); + this.description = description; this.enumValues = ParamChecker.notNullElements(enumValues, "Enum values"); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java index 5a331a6..7c8db84 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java @@ -49,7 +49,6 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A public final TypeSystem typeSystem; public final Class<ST> superTypeClass; - public final String name; public final FieldMapping fieldMapping; public final int numFields; public final ImmutableList<String> superTypes; @@ -58,14 +57,19 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A protected ImmutableMap<String, List<Path>> superTypePaths; protected ImmutableMap<String, Path> pathNameToPathMap; + HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, ImmutableList<String> superTypes, + int numFields) { + this(typeSystem, superTypeClass, name, null, superTypes, numFields); + } + /** * Used when creating a Type, to support recursive Structs. */ - HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, ImmutableList<String> superTypes, + HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, String description, ImmutableList<String> superTypes, int numFields) { + super(name, description); this.typeSystem = typeSystem; this.superTypeClass = superTypeClass; - this.name = name; this.fieldMapping = null; this.numFields = numFields; this.superTypes = superTypes; @@ -74,10 +78,14 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A } HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, ImmutableList<String> superTypes, + AttributeInfo... fields) throws AtlasException { + this(typeSystem, superTypeClass, name, null, superTypes, fields); + } + HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass, String name, String description, ImmutableList<String> superTypes, AttributeInfo... fields) throws AtlasException { + super(name, description); this.typeSystem = typeSystem; this.superTypeClass = superTypeClass; - this.name = name; Pair<FieldMapping, ImmutableMap<String, String>> p = constructFieldMapping(superTypes, fields); this.fieldMapping = p.left; this.attributeNameToType = p.right; @@ -86,11 +94,6 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A this.immediateAttrs = ImmutableList.copyOf(fields); } - @Override - public String getName() { - return name; - } - public FieldMapping fieldMapping() { return fieldMapping; } @@ -490,6 +493,4 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A throw new UnsupportedOperationException(); } } - - } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalTypeDefinition.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalTypeDefinition.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalTypeDefinition.java index 3188840..973a591 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalTypeDefinition.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalTypeDefinition.java @@ -31,20 +31,21 @@ public class HierarchicalTypeDefinition<T extends HierarchicalType> extends Stru * not intended public consumption * @param hierarchicalMetaTypeName * @param typeName + * @param typeDescription * @param superTypes * @param attributeDefinitions * @throws ClassNotFoundException */ @InterfaceAudience.Private - public HierarchicalTypeDefinition(String hierarchicalMetaTypeName, String typeName, String[] superTypes, + public HierarchicalTypeDefinition(String hierarchicalMetaTypeName, String typeName, String typeDescription, String[] superTypes, AttributeDefinition[] attributeDefinitions) throws ClassNotFoundException { - this((Class<T>) Class.forName(hierarchicalMetaTypeName), typeName, ImmutableList.copyOf(superTypes), + this((Class<T>) Class.forName(hierarchicalMetaTypeName), typeName, typeDescription, ImmutableList.copyOf(superTypes), attributeDefinitions); } - public HierarchicalTypeDefinition(Class<T> hierarchicalMetaType, String typeName, ImmutableList<String> superTypes, - AttributeDefinition[] attributeDefinitions) { - super(typeName, false, attributeDefinitions); + public HierarchicalTypeDefinition(Class<T> hierarchicalMetaType, String typeName, String typeDescription, ImmutableList<String> superTypes, + AttributeDefinition[] attributeDefinitions) { + super(typeName, typeDescription, false, attributeDefinitions); hierarchicalMetaTypeName = hierarchicalMetaType.getName(); this.superTypes = superTypes == null ? ImmutableList.<String>of() : superTypes; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java index 293014e..373ad2c 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java @@ -34,5 +34,7 @@ public interface IDataType<T> { void validateUpdate(IDataType newType) throws TypeUpdateException; void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException; + + String getDescription(); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java index ba053d2..54e344f 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java @@ -18,10 +18,6 @@ package org.apache.atlas.typesystem.types; -import org.apache.atlas.AtlasException; -import org.apache.atlas.typesystem.IStruct; -import org.apache.atlas.typesystem.ITypedStruct; - import java.nio.charset.Charset; import java.security.MessageDigest; import java.util.HashMap; @@ -29,31 +25,31 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import org.apache.atlas.AtlasException; +import org.apache.atlas.typesystem.IStruct; +import org.apache.atlas.typesystem.ITypedStruct; + public class StructType extends AbstractDataType<IStruct> implements IConstructableType<IStruct, ITypedStruct> { public final TypeSystem typeSystem; - public final String name; public final FieldMapping fieldMapping; public final Map<AttributeInfo, List<String>> infoToNameMap; public final int numFields; private final TypedStructHandler handler; - /** - * Used when creating a StructType, to support recursive Structs. - */ - protected StructType(TypeSystem typeSystem, String name, int numFields) { + protected StructType(TypeSystem typeSystem, String name, String description, int numFields) { + super(name, description); this.typeSystem = typeSystem; - this.name = name; this.fieldMapping = null; infoToNameMap = null; this.numFields = numFields; this.handler = null; } - protected StructType(TypeSystem typeSystem, String name, AttributeInfo... fields) + protected StructType(TypeSystem typeSystem, String name, String description, AttributeInfo... fields) throws AtlasException { + super(name, description); this.typeSystem = typeSystem; - this.name = name; this.fieldMapping = constructFieldMapping(fields); infoToNameMap = TypeUtils.buildAttrInfoToNameMap(this.fieldMapping); this.numFields = this.fieldMapping.fields.size(); @@ -64,11 +60,6 @@ public class StructType extends AbstractDataType<IStruct> implements IConstructa return fieldMapping; } - @Override - public String getName() { - return name; - } - /** * Validate that current definition can be updated with the new definition * @param newType @@ -216,4 +207,5 @@ public class StructType extends AbstractDataType<IStruct> implements IConstructa public List<String> getNames(AttributeInfo info) { return infoToNameMap.get(info); } + } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java index e47c4e5..9dc93ae 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java @@ -25,10 +25,16 @@ import java.util.Arrays; public class StructTypeDefinition { public final String typeName; + public final String typeDescription;//optional field public final AttributeDefinition[] attributeDefinitions; protected StructTypeDefinition(String typeName, boolean validate, AttributeDefinition... attributeDefinitions) { + this(typeName, null, validate, attributeDefinitions); + } + + protected StructTypeDefinition(String typeName, String typeDescription, boolean validate, AttributeDefinition... attributeDefinitions) { this.typeName = ParamChecker.notEmpty(typeName, "Struct type name"); + this.typeDescription = typeDescription; if (attributeDefinitions != null && attributeDefinitions.length != 0) { ParamChecker.notNullElements(attributeDefinitions, "Attribute definitions"); } @@ -36,7 +42,13 @@ public class StructTypeDefinition { } public StructTypeDefinition(String typeName, AttributeDefinition[] attributeDefinitions) { + this(typeName, null, attributeDefinitions); + } + + public StructTypeDefinition(String typeName, String typeDescription, + AttributeDefinition[] attributeDefinitions) { this.typeName = ParamChecker.notEmpty(typeName, "Struct type name"); + this.typeDescription = typeDescription; this.attributeDefinitions = ParamChecker.notNullElements(attributeDefinitions, "Attribute definitions"); } @@ -57,7 +69,6 @@ public class StructTypeDefinition { if (!typeName.equals(that.typeName)) { return false; } - return true; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java index f5a3875..baf5b4d 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java @@ -34,18 +34,15 @@ public class TraitType extends HierarchicalType<TraitType, IStruct> public final Map<AttributeInfo, List<String>> infoToNameMap; private final TypedStructHandler handler; - /** - * Used when creating a TraitType, to support recursive Structs. - */ - TraitType(TypeSystem typeSystem, String name, ImmutableList<String> superTraits, int numFields) { - super(typeSystem, TraitType.class, name, superTraits, numFields); + TraitType(TypeSystem typeSystem, String name, String description, ImmutableList<String> superTraits, int numFields) { + super(typeSystem, TraitType.class, name, description, superTraits, numFields); handler = null; infoToNameMap = null; } - TraitType(TypeSystem typeSystem, String name, ImmutableList<String> superTraits, AttributeInfo... fields) + TraitType(TypeSystem typeSystem, String name, String description, ImmutableList<String> superTraits, AttributeInfo... fields) throws AtlasException { - super(typeSystem, TraitType.class, name, superTraits, fields); + super(typeSystem, TraitType.class, name, description, superTraits, fields); handler = new TypedStructHandler(this); infoToNameMap = TypeUtils.buildAttrInfoToNameMap(fieldMapping); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java index c0b0698..bd81d7f 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java @@ -168,7 +168,12 @@ public class TypeSystem { public StructType defineStructType(String name, boolean errorIfExists, AttributeDefinition... attrDefs) throws AtlasException { - StructTypeDefinition structDef = new StructTypeDefinition(name, attrDefs); + return defineStructType(name, null, errorIfExists, attrDefs); + } + + public StructType defineStructType(String name, String description, boolean errorIfExists, AttributeDefinition... attrDefs) + throws AtlasException { + StructTypeDefinition structDef = new StructTypeDefinition(name, description, attrDefs); defineTypes(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.of(structDef), ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), ImmutableList.<HierarchicalTypeDefinition<ClassType>>of()); @@ -192,7 +197,7 @@ public class TypeSystem { infos[i] = new AttributeInfo(this, attrDefs[i], tempTypes); } - return new StructType(this, name, infos); + return new StructType(this, name, null, infos); } public TraitType defineTraitType(HierarchicalTypeDefinition<TraitType> traitDef) throws AtlasException { @@ -272,13 +277,17 @@ public class TypeSystem { return defineEnumType(new EnumTypeDefinition(name, values)); } + public EnumType defineEnumType(String name, String description, EnumValue... values) throws AtlasException { + return defineEnumType(new EnumTypeDefinition(name, description, values)); + } + public EnumType defineEnumType(EnumTypeDefinition eDef) throws AtlasException { assert eDef.name != null; if (types.containsKey(eDef.name)) { throw new AtlasException(String.format("Redefinition of type %s not supported", eDef.name)); } - EnumType eT = new EnumType(this, eDef.name, eDef.enumValues); + EnumType eT = new EnumType(this, eDef.name, eDef.description, eDef.enumValues); types.put(eDef.name, eT); typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.ENUM, eDef.name); return eT; @@ -348,7 +357,7 @@ public class TypeSystem { throw new AtlasException(String.format("Redefinition of type %s not supported", eDef.name)); } - EnumType eT = new EnumType(this, eDef.name, eDef.enumValues); + EnumType eT = new EnumType(this, eDef.name, eDef.description, eDef.enumValues); transientTypes.put(eDef.name, eT); } @@ -357,7 +366,7 @@ public class TypeSystem { if (!update && (transientTypes.containsKey(sDef.typeName) || types.containsKey(sDef.typeName))) { throw new TypeExistsException(String.format("Cannot redefine type %s", sDef.typeName)); } - StructType sT = new StructType(this, sDef.typeName, sDef.attributeDefinitions.length); + StructType sT = new StructType(this, sDef.typeName, sDef.typeDescription, sDef.attributeDefinitions.length); structNameToDefMap.put(sDef.typeName, sDef); transientTypes.put(sDef.typeName, sT); } @@ -368,7 +377,7 @@ public class TypeSystem { (transientTypes.containsKey(traitDef.typeName) || types.containsKey(traitDef.typeName))) { throw new TypeExistsException(String.format("Cannot redefine type %s", traitDef.typeName)); } - TraitType tT = new TraitType(this, traitDef.typeName, traitDef.superTypes, + TraitType tT = new TraitType(this, traitDef.typeName, traitDef.typeDescription, traitDef.superTypes, traitDef.attributeDefinitions.length); traitNameToDefMap.put(traitDef.typeName, traitDef); transientTypes.put(traitDef.typeName, tT); @@ -381,7 +390,7 @@ public class TypeSystem { throw new TypeExistsException(String.format("Cannot redefine type %s", classDef.typeName)); } - ClassType cT = new ClassType(this, classDef.typeName, classDef.superTypes, + ClassType cT = new ClassType(this, classDef.typeName, classDef.typeDescription, classDef.superTypes, classDef.attributeDefinitions.length); classNameToDefMap.put(classDef.typeName, classDef); transientTypes.put(classDef.typeName, cT); @@ -471,7 +480,7 @@ public class TypeSystem { infos[i] = constructAttributeInfo(def.attributeDefinitions[i]); } - StructType type = new StructType(this, def.typeName, infos); + StructType type = new StructType(this, def.typeName, def.typeDescription, infos); transientTypes.put(def.typeName, type); return type; } @@ -484,9 +493,9 @@ public class TypeSystem { } try { - Constructor<U> cons = cls.getDeclaredConstructor(TypeSystem.class, String.class, ImmutableList.class, + Constructor<U> cons = cls.getDeclaredConstructor(TypeSystem.class, String.class, String.class, ImmutableList.class, AttributeInfo[].class); - U type = cons.newInstance(this, def.typeName, def.superTypes, infos); + U type = cons.newInstance(this, def.typeName, def.typeDescription, def.superTypes, infos); transientTypes.put(def.typeName, type); return type; } catch (Exception e) { @@ -682,7 +691,7 @@ public class TypeSystem { infos[0] = new AttributeInfo(TypeSystem.this, idAttr, null); infos[1] = new AttributeInfo(TypeSystem.this, typNmAttr, null); - StructType type = new StructType(TypeSystem.this, TYP_NAME, infos); + StructType type = new StructType(TypeSystem.this, TYP_NAME, null, infos); TypeSystem.this.types.put(TYP_NAME, type); } catch (AtlasException me) { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java index ee82ce5..80968d8 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java @@ -65,16 +65,30 @@ public class TypesUtil { public static HierarchicalTypeDefinition<TraitType> createTraitTypeDef(String name, ImmutableList<String> superTypes, AttributeDefinition... attrDefs) { - return new HierarchicalTypeDefinition<>(TraitType.class, name, superTypes, attrDefs); + return createTraitTypeDef(name, null, superTypes, attrDefs); + } + + public static HierarchicalTypeDefinition<TraitType> createTraitTypeDef(String name, String description, + ImmutableList<String> superTypes, AttributeDefinition... attrDefs) { + return new HierarchicalTypeDefinition<>(TraitType.class, name, description, superTypes, attrDefs); } public static StructTypeDefinition createStructTypeDef(String name, AttributeDefinition... attrDefs) { - return new StructTypeDefinition(name, attrDefs); + return createStructTypeDef(name, null, attrDefs); + } + + public static StructTypeDefinition createStructTypeDef(String name, String description, AttributeDefinition... attrDefs) { + return new StructTypeDefinition(name, description, attrDefs); } public static HierarchicalTypeDefinition<ClassType> createClassTypeDef(String name, ImmutableList<String> superTypes, AttributeDefinition... attrDefs) { - return new HierarchicalTypeDefinition<>(ClassType.class, name, superTypes, attrDefs); + return createClassTypeDef(name, null, superTypes, attrDefs); + } + + public static HierarchicalTypeDefinition<ClassType> createClassTypeDef(String name, String description, + ImmutableList<String> superTypes, AttributeDefinition... attrDefs) { + return new HierarchicalTypeDefinition<>(ClassType.class, name, description, superTypes, attrDefs); } public static TypesDef getTypesDef(ImmutableList<EnumTypeDefinition> enums, http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/typesystem/src/main/scala/org/apache/atlas/typesystem/json/TypesSerialization.scala ---------------------------------------------------------------------- diff --git a/typesystem/src/main/scala/org/apache/atlas/typesystem/json/TypesSerialization.scala b/typesystem/src/main/scala/org/apache/atlas/typesystem/json/TypesSerialization.scala index 6bdbc79..5d50c45 100755 --- a/typesystem/src/main/scala/org/apache/atlas/typesystem/json/TypesSerialization.scala +++ b/typesystem/src/main/scala/org/apache/atlas/typesystem/json/TypesSerialization.scala @@ -131,28 +131,28 @@ object TypesSerialization { private def convertEnumTypeToEnumTypeDef(et: EnumType) = { val eVals: Seq[EnumValue] = et.valueMap.values().toSeq - new EnumTypeDefinition(et.name, eVals: _*) + new EnumTypeDefinition(et.name, et.description, eVals: _*) } private def convertStructTypeToStructDef(st: StructType): StructTypeDefinition = { val aDefs: Iterable[AttributeDefinition] = st.fieldMapping.fields.values().map(convertAttributeInfoToAttributeDef(_)) - new StructTypeDefinition(st.name, aDefs.toArray) + new StructTypeDefinition(st.name, st.description, aDefs.toArray) } private def convertTraitTypeToHierarchicalTypeDefintion(tt: TraitType): HierarchicalTypeDefinition[TraitType] = { val aDefs: Iterable[AttributeDefinition] = tt.immediateAttrs.map(convertAttributeInfoToAttributeDef(_)) - new HierarchicalTypeDefinition[TraitType](classOf[TraitType], tt.name, tt.superTypes, aDefs.toArray) + new HierarchicalTypeDefinition[TraitType](classOf[TraitType], tt.name, tt.description, tt.superTypes, aDefs.toArray) } private def convertClassTypeToHierarchicalTypeDefintion(tt: ClassType): HierarchicalTypeDefinition[ClassType] = { val aDefs: Iterable[AttributeDefinition] = tt.immediateAttrs.map(convertAttributeInfoToAttributeDef(_)) - new HierarchicalTypeDefinition[ClassType](classOf[ClassType], tt.name, tt.superTypes, aDefs.toArray) + new HierarchicalTypeDefinition[ClassType](classOf[ClassType], tt.name, tt.description, tt.superTypes, aDefs.toArray) } def convertToTypesDef(ts: TypeSystem, export: IDataType[_] => Boolean): TypesDef = { @@ -212,8 +212,13 @@ trait TypeHelpers { def optionalAttr(name: String, dataType: IDataType[_]) = new AttributeDefinition(name, dataType.getName, Multiplicity.OPTIONAL, false, null) - def structDef(name: String, attrs: AttributeDefinition*) = { - new StructTypeDefinition(name, attrs.toArray) + def structDef(name: String, attrs: AttributeDefinition*): + StructTypeDefinition = { + structDef(name, None, attrs:_*) + } + + def structDef(name: String, description: Option[String], attrs: AttributeDefinition*) = { + new StructTypeDefinition(name, description.getOrElse(null), attrs.toArray) } def defineTraits(ts: TypeSystem, tDefs: HierarchicalTypeDefinition[TraitType]*) = { @@ -222,15 +227,25 @@ trait TypeHelpers { def createTraitTypeDef(name: String, superTypes: Seq[String], attrDefs: AttributeDefinition*): HierarchicalTypeDefinition[TraitType] = { + createTraitTypeDef(name, None, superTypes, attrDefs:_*) + } + + def createTraitTypeDef(name: String, description: Option[String], superTypes: Seq[String], attrDefs: AttributeDefinition*): + HierarchicalTypeDefinition[TraitType] = { val sts = ImmutableList.copyOf(superTypes.toArray) - return new HierarchicalTypeDefinition[TraitType](classOf[TraitType], name, + return new HierarchicalTypeDefinition[TraitType](classOf[TraitType], name, description.getOrElse(null), sts, attrDefs.toArray) } def createClassTypeDef(name: String, superTypes: Seq[String], attrDefs: AttributeDefinition*): HierarchicalTypeDefinition[ClassType] = { + createClassTypeDef( name, None, superTypes, attrDefs:_*) + } + + def createClassTypeDef(name: String, description: Option[String], superTypes: Seq[String], attrDefs: AttributeDefinition*): + HierarchicalTypeDefinition[ClassType] = { val sts = ImmutableList.copyOf(superTypes.toArray) - return new HierarchicalTypeDefinition[ClassType](classOf[ClassType], name, + return new HierarchicalTypeDefinition[ClassType](classOf[ClassType], name, description.getOrElse(null), sts, attrDefs.toArray) } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6fb1f1a/typesystem/src/test/java/org/apache/atlas/typesystem/types/ClassTest.java ---------------------------------------------------------------------- diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/ClassTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/ClassTest.java index f454641..a0f5843 100755 --- a/typesystem/src/test/java/org/apache/atlas/typesystem/types/ClassTest.java +++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/ClassTest.java @@ -18,16 +18,18 @@ package org.apache.atlas.typesystem.types; -import com.google.common.collect.ImmutableList; import org.apache.atlas.AtlasException; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.TypesDef; +import org.apache.atlas.typesystem.json.TypesSerialization; import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.testng.Assert; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import com.google.common.collect.ImmutableList; + public class ClassTest extends HierarchicalTypeTest<ClassType> { @BeforeMethod @@ -72,15 +74,38 @@ public class ClassTest extends HierarchicalTypeTest<ClassType> { "}"); } + + @Test + public void testSerDeWithoutDescription() throws Exception { + HierarchicalTypeDefinition<ClassType> clsType = TypesUtil + .createClassTypeDef("Random", ImmutableList.<String>of(), + TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE)); + + TypesDef typesDef = getTypesDef(clsType); + String json = TypesSerialization.toJson(typesDef); + System.out.println("json " + json); + TypesSerialization.fromJson(json); + } + + @Test + public void testSerDeWithDescription() throws Exception { + HierarchicalTypeDefinition<ClassType> clsType = TypesUtil + .createClassTypeDef("Random", "Random-description", ImmutableList.<String>of(), + TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE)); + TypesDef typesDef = getTypesDef(clsType); + String json = TypesSerialization.toJson(typesDef); + System.out.println("json " + json); + TypesSerialization.fromJson(json); + } @Override protected HierarchicalTypeDefinition<ClassType> getTypeDefinition(String name, AttributeDefinition... attributes) { - return new HierarchicalTypeDefinition(ClassType.class, name, null, attributes); + return new HierarchicalTypeDefinition(ClassType.class, name, null, null, attributes); } @Override protected HierarchicalTypeDefinition<ClassType> getTypeDefinition(String name, ImmutableList<String> superTypes, AttributeDefinition... attributes) { - return new HierarchicalTypeDefinition(ClassType.class, name, superTypes, attributes); + return new HierarchicalTypeDefinition(ClassType.class, name, null, superTypes, attributes); } @Override
