Repository: atlas Updated Branches: refs/heads/master 9a5553731 -> e4921452e
ATLAS-2984: export should include type defintions of relationships referenced by entities Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/e4921452 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/e4921452 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/e4921452 Branch: refs/heads/master Commit: e4921452ebea5b338c07bbc6f886c44c8843f64b Parents: 9a55537 Author: Madhan Neethiraj <mad...@apache.org> Authored: Thu Nov 29 13:10:58 2018 -0800 Committer: Madhan Neethiraj <mad...@apache.org> Committed: Thu Nov 29 17:33:45 2018 -0800 ---------------------------------------------------------------------- .../atlas/repository/impexp/ExportService.java | 30 ++++++++++++++++++++ .../repository/impexp/ExportTypeProcessor.java | 24 ++++++++++++++++ .../impexp/ImportTypeDefProcessor.java | 1 + 3 files changed, 55 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/e4921452/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java index 3558d2a..5632520 100644 --- a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java +++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java @@ -32,6 +32,7 @@ import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import org.apache.atlas.model.typedef.AtlasClassificationDef; import org.apache.atlas.model.typedef.AtlasEntityDef; import org.apache.atlas.model.typedef.AtlasEnumDef; +import org.apache.atlas.model.typedef.AtlasRelationshipDef; import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.atlas.model.typedef.AtlasTypesDef; @@ -43,6 +44,7 @@ import org.apache.atlas.type.AtlasClassificationType; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasEnumType; import org.apache.atlas.type.AtlasMapType; +import org.apache.atlas.type.AtlasRelationshipType; import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.type.AtlasStructType.AtlasAttribute; import org.apache.atlas.type.AtlasType; @@ -175,6 +177,12 @@ public class ExportService { typesDef.getEnumDefs().add(enumDef); } + + for (String relationshipType : context.relationshipTypes) { + AtlasRelationshipDef relationshipDef = typeRegistry.getRelationshipDefByName(relationshipType); + + typesDef.getRelationshipDefs().add(relationshipDef); + } } private AtlasExportResult.OperationStatus[] processItems(AtlasExportRequest request, ExportContext context) { @@ -611,6 +619,8 @@ public class ExportService { addStructType((AtlasStructType)type, context); } else if (type instanceof AtlasEnumType) { addEnumType((AtlasEnumType)type, context); + } else if (type instanceof AtlasRelationshipType) { + addRelationshipType((AtlasRelationshipType)type, context); } } @@ -619,6 +629,7 @@ public class ExportService { context.entityTypes.add(entityType.getTypeName()); addAttributeTypes(entityType, context); + addRelationshipTypes(entityType, context); if (CollectionUtils.isNotEmpty(entityType.getAllSuperTypes())) { for (String superType : entityType.getAllSuperTypes()) { @@ -656,12 +667,30 @@ public class ExportService { } } + private void addRelationshipType(AtlasRelationshipType relationshipType, ExportContext context) { + if (!context.relationshipTypes.contains(relationshipType.getTypeName())) { + context.relationshipTypes.add(relationshipType.getTypeName()); + + addAttributeTypes(relationshipType, context); + addEntityType(relationshipType.getEnd1Type(), context); + addEntityType(relationshipType.getEnd2Type(), context); + } + } + private void addAttributeTypes(AtlasStructType structType, ExportContext context) { for (AtlasAttributeDef attributeDef : structType.getStructDef().getAttributeDefs()) { addType(attributeDef.getTypeName(), context); } } + private void addRelationshipTypes(AtlasEntityType entityType, ExportContext context) { + for (List<AtlasRelationshipType> relationshipTypes : entityType.getRelationshipAttributesType().values()) { + for (AtlasRelationshipType relationshipType : relationshipTypes) { + addRelationshipType(relationshipType, context); + } + } + } + private List<Map<String, Object>> executeGremlinQuery(String query, ExportContext context) { try { return (List<Map<String, Object>>) atlasGraph.executeGremlinScript(context.scriptEngine, context.bindings, query, false); @@ -724,6 +753,7 @@ public class ExportService { final Set<String> classificationTypes = new HashSet<>(); final Set<String> structTypes = new HashSet<>(); final Set<String> enumTypes = new HashSet<>(); + final Set<String> relationshipTypes = new HashSet<>(); final AtlasExportResult result; private final ZipSink sink; http://git-wip-us.apache.org/repos/asf/atlas/blob/e4921452/repository/src/main/java/org/apache/atlas/repository/impexp/ExportTypeProcessor.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportTypeProcessor.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportTypeProcessor.java index 6b5db61..f528b17 100644 --- a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportTypeProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportTypeProcessor.java @@ -28,6 +28,7 @@ import org.apache.atlas.type.AtlasClassificationType; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasEnumType; import org.apache.atlas.type.AtlasMapType; +import org.apache.atlas.type.AtlasRelationshipType; import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; @@ -35,6 +36,8 @@ import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; + class ExportTypeProcessor { private static final Logger LOG = LoggerFactory.getLogger(ExportTypeProcessor.class); @@ -106,6 +109,8 @@ class ExportTypeProcessor { addStructType((AtlasStructType)type, context); } else if (type instanceof AtlasEnumType) { addEnumType((AtlasEnumType)type, context); + } else if (type instanceof AtlasRelationshipType) { + addRelationshipType((AtlasRelationshipType)type, context); } } @@ -114,6 +119,7 @@ class ExportTypeProcessor { context.entityTypes.add(entityType.getTypeName()); addAttributeTypes(entityType, context); + addRelationshipTypes(entityType, context); if (CollectionUtils.isNotEmpty(entityType.getAllSuperTypes())) { for (String superType : entityType.getAllSuperTypes()) { @@ -151,9 +157,27 @@ class ExportTypeProcessor { } } + private void addRelationshipType(AtlasRelationshipType relationshipType, ExportService.ExportContext context) { + if (!context.relationshipTypes.contains(relationshipType.getTypeName())) { + context.relationshipTypes.add(relationshipType.getTypeName()); + + addAttributeTypes(relationshipType, context); + addEntityType(relationshipType.getEnd1Type(), context); + addEntityType(relationshipType.getEnd2Type(), context); + } + } + private void addAttributeTypes(AtlasStructType structType, ExportService.ExportContext context) { for (AtlasStructDef.AtlasAttributeDef attributeDef : structType.getStructDef().getAttributeDefs()) { addType(attributeDef.getTypeName(), context); } } + + private void addRelationshipTypes(AtlasEntityType entityType, ExportService.ExportContext context) { + for (List<AtlasRelationshipType> relationshipTypes : entityType.getRelationshipAttributesType().values()) { + for (AtlasRelationshipType relationshipType : relationshipTypes) { + addRelationshipType(relationshipType, context); + } + } + } } http://git-wip-us.apache.org/repos/asf/atlas/blob/e4921452/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTypeDefProcessor.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTypeDefProcessor.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTypeDefProcessor.java index ab66460..c217937 100644 --- a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTypeDefProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTypeDefProcessor.java @@ -75,5 +75,6 @@ public class ImportTypeDefProcessor { result.incrementMeticsCounter("typedef:enum", typeDefinitionMap.getEnumDefs().size()); result.incrementMeticsCounter("typedef:entitydef", typeDefinitionMap.getEntityDefs().size()); result.incrementMeticsCounter("typedef:struct", typeDefinitionMap.getStructDefs().size()); + result.incrementMeticsCounter("typedef:relationship", typeDefinitionMap.getRelationshipDefs().size()); } }
