This is an automated email from the ASF dual-hosted git repository.

amestry pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git


The following commit(s) were added to refs/heads/master by this push:
     new 49db4ca  ATLAS-3176: Fix for export order regression.
49db4ca is described below

commit 49db4cacbb07dd018876e77b872f483ef941a152
Author: Ashutosh Mestry <[email protected]>
AuthorDate: Fri May 10 15:35:57 2019 -0700

    ATLAS-3176: Fix for export order regression.
---
 .../atlas/repository/impexp/ExportService.java     | 182 ++-------------------
 .../repository/impexp/ExportTypeProcessor.java     |   4 +-
 .../apache/atlas/repository/impexp/ZipSource.java  |   8 +-
 .../web/resources/AdminExportImportTestIT.java     |   4 +-
 4 files changed, 23 insertions(+), 175 deletions(-)

diff --git 
a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
 
b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
index 9fdfb0f..11289ea 100644
--- 
a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
+++ 
b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
@@ -20,10 +20,8 @@ package org.apache.atlas.repository.impexp;
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.atlas.RequestContext;
 import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.model.TypeCategory;
 import org.apache.atlas.model.impexp.AtlasExportRequest;
 import org.apache.atlas.model.impexp.AtlasExportResult;
-import org.apache.atlas.model.instance.AtlasClassification;
 import org.apache.atlas.model.instance.AtlasEntity;
 import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
 import org.apache.atlas.model.instance.AtlasObjectId;
@@ -33,20 +31,11 @@ import org.apache.atlas.model.typedef.AtlasEntityDef;
 import org.apache.atlas.model.typedef.AtlasEnumDef;
 import org.apache.atlas.model.typedef.AtlasRelationshipDef;
 import org.apache.atlas.model.typedef.AtlasStructDef;
-import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
 import org.apache.atlas.model.typedef.AtlasTypesDef;
 import org.apache.atlas.repository.graphdb.AtlasGraph;
 import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
 import org.apache.atlas.repository.util.UniqueList;
-import org.apache.atlas.type.AtlasArrayType;
-import org.apache.atlas.type.AtlasClassificationType;
 import org.apache.atlas.type.AtlasEntityType;
-import org.apache.atlas.type.AtlasEnumType;
-import org.apache.atlas.type.AtlasMapType;
-import org.apache.atlas.type.AtlasRelationshipType;
-import org.apache.atlas.type.AtlasStructType;
-import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
-import org.apache.atlas.type.AtlasType;
 import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.atlas.type.AtlasTypeUtil;
 import org.apache.atlas.util.AtlasGremlinQueryProvider;
@@ -59,9 +48,16 @@ import org.springframework.stereotype.Component;
 import javax.inject.Inject;
 import javax.script.ScriptEngine;
 import javax.script.ScriptException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
-import static org.apache.atlas.model.impexp.AtlasExportRequest.*;
+import static 
org.apache.atlas.model.impexp.AtlasExportRequest.FETCH_TYPE_CONNECTED;
+import static org.apache.atlas.model.impexp.AtlasExportRequest.FETCH_TYPE_FULL;
+import static 
org.apache.atlas.model.impexp.AtlasExportRequest.FETCH_TYPE_INCREMENTAL;
 
 @Component
 public class ExportService {
@@ -70,7 +66,6 @@ public class ExportService {
     public static final String PROPERTY_GUID = "__guid";
     private static final String PROPERTY_IS_PROCESS = "isProcess";
 
-
     private final AtlasTypeRegistry         typeRegistry;
     private final String QUERY_BINDING_START_GUID = "startGuid";
     private final StartEntityFetchByExportRequest 
startEntityFetchByExportRequest;
@@ -101,7 +96,7 @@ public class ExportService {
                 hostName, startTime, getCurrentChangeMarker());
 
         ExportContext context = new ExportContext(atlasGraph, result, 
exportSink);
-        exportTypeProcessor = new ExportTypeProcessor(typeRegistry, context);
+        exportTypeProcessor = new ExportTypeProcessor(typeRegistry);
 
         try {
             LOG.info("==> export(user={}, from={})", userName, requestingIP);
@@ -134,7 +129,6 @@ public class ExportService {
                                                 long startTime, long endTime) 
throws AtlasBaseException {
         int duration = getOperationDuration(startTime, endTime);
         
context.result.setSourceClusterName(AuditsWriter.getCurrentClusterName());
-        context.addToEntityCreationOrder(context.lineageProcessed);
 
         context.sink.setExportOrder(context.entityCreationOrder.getList());
         context.sink.setTypesDef(context.result.getData().getTypesDef());
@@ -269,18 +263,14 @@ public class ExportService {
         TraversalDirection direction = context.guidDirection.get(guid);
         AtlasEntityWithExtInfo entityWithExtInfo = 
entityGraphRetriever.toAtlasEntityWithExtInfo(guid);
 
-        processEntity(guid, entityWithExtInfo, context, direction);
+        processEntity(entityWithExtInfo, context, direction);
 
         debugLog("<== processEntityGuid({})", guid);
     }
 
-    public void processEntity(String guid, AtlasEntityWithExtInfo 
entityWithExtInfo,
-                               ExportContext context,
-                               TraversalDirection direction) throws 
AtlasBaseException {
-
-        if (!context.lineageProcessed.contains(guid) && 
context.doesTimestampQualify(entityWithExtInfo.getEntity())) {
-            
context.addToEntityCreationOrder(entityWithExtInfo.getEntity().getGuid());
-        }
+    public void processEntity(AtlasEntityWithExtInfo entityWithExtInfo,
+                              ExportContext context,
+                              TraversalDirection direction) throws 
AtlasBaseException {
 
         addEntity(entityWithExtInfo, context);
         exportTypeProcessor.addTypes(entityWithExtInfo.getEntity(), context);
@@ -315,7 +305,7 @@ public class ExportService {
         }
     }
 
-    private void populateEntitesForIncremental(String topLevelEntityGuid, 
ExportContext context) throws AtlasBaseException {
+    private void populateEntitesForIncremental(String topLevelEntityGuid, 
ExportContext context) {
         if (context.isHiveDBIncrementalSkipLineage() == false || 
incrementalExportEntityProvider != null) {
             return;
         }
@@ -450,7 +440,6 @@ public class ExportService {
         } else {
             List<AtlasEntity> entities = 
context.getEntitiesWithModifiedTimestamp(entityWithExtInfo);
             for (AtlasEntity e : entities) {
-                context.addToEntityCreationOrder(e.getGuid());
                 context.addToSink(new AtlasEntityWithExtInfo(e));
                 
context.result.incrementMeticsCounter(String.format("entity:%s", 
e.getTypeName()));
             }
@@ -459,142 +448,6 @@ public class ExportService {
         context.reportProgress();
     }
 
-    private void addTypes(AtlasEntity entity, ExportContext context) {
-        addEntityType(entity.getTypeName(), context);
-
-        if(CollectionUtils.isNotEmpty(entity.getClassifications())) {
-            for (AtlasClassification c : entity.getClassifications()) {
-                addClassificationType(c.getTypeName(), context);
-            }
-        }
-    }
-
-    private void addType(String typeName, ExportContext context) {
-        AtlasType type = null;
-
-        try {
-            type = typeRegistry.getType(typeName);
-
-            addType(type, context);
-        } catch (AtlasBaseException excp) {
-            LOG.error("unknown type {}", typeName);
-        }
-    }
-
-    private void addEntityType(String typeName, ExportContext context) {
-        if (!context.entityTypes.contains(typeName)) {
-            AtlasEntityType entityType = 
typeRegistry.getEntityTypeByName(typeName);
-
-            addEntityType(entityType, context);
-        }
-    }
-
-    private void addClassificationType(String typeName, ExportContext context) 
{
-        if (!context.classificationTypes.contains(typeName)) {
-            AtlasClassificationType classificationType = 
typeRegistry.getClassificationTypeByName(typeName);
-
-            addClassificationType(classificationType, context);
-        }
-    }
-
-    private void addType(AtlasType type, ExportContext context) {
-        if (type.getTypeCategory() == TypeCategory.PRIMITIVE) {
-            return;
-        }
-
-        if (type instanceof AtlasArrayType) {
-            AtlasArrayType arrayType = (AtlasArrayType)type;
-
-            addType(arrayType.getElementType(), context);
-        } else if (type instanceof AtlasMapType) {
-            AtlasMapType mapType = (AtlasMapType)type;
-
-            addType(mapType.getKeyType(), context);
-            addType(mapType.getValueType(), context);
-        } else if (type instanceof AtlasEntityType) {
-            addEntityType((AtlasEntityType)type, context);
-        } else if (type instanceof AtlasClassificationType) {
-            addClassificationType((AtlasClassificationType)type, context);
-        } else if (type instanceof AtlasStructType) {
-            addStructType((AtlasStructType)type, context);
-        } else if (type instanceof AtlasEnumType) {
-            addEnumType((AtlasEnumType)type, context);
-        } else if (type instanceof AtlasRelationshipType) {
-            addRelationshipType(type.getTypeName(), context);
-        }
-    }
-
-    private void addEntityType(AtlasEntityType entityType, ExportContext 
context) {
-        if (!context.entityTypes.contains(entityType.getTypeName())) {
-            context.entityTypes.add(entityType.getTypeName());
-
-            addAttributeTypes(entityType, context);
-            addRelationshipTypes(entityType, context);
-
-            if (CollectionUtils.isNotEmpty(entityType.getAllSuperTypes())) {
-                for (String superType : entityType.getAllSuperTypes()) {
-                    addEntityType(superType, context);
-                }
-            }
-        }
-    }
-
-    private void addClassificationType(AtlasClassificationType 
classificationType, ExportContext context) {
-        if 
(!context.classificationTypes.contains(classificationType.getTypeName())) {
-            context.classificationTypes.add(classificationType.getTypeName());
-
-            addAttributeTypes(classificationType, context);
-
-            if 
(CollectionUtils.isNotEmpty(classificationType.getAllSuperTypes())) {
-                for (String superType : classificationType.getAllSuperTypes()) 
{
-                    addClassificationType(superType, context);
-                }
-            }
-        }
-    }
-
-    private void addStructType(AtlasStructType structType, ExportContext 
context) {
-        if (!context.structTypes.contains(structType.getTypeName())) {
-            context.structTypes.add(structType.getTypeName());
-
-            addAttributeTypes(structType, context);
-        }
-    }
-
-    private void addEnumType(AtlasEnumType enumType, ExportContext context) {
-        if (!context.enumTypes.contains(enumType.getTypeName())) {
-            context.enumTypes.add(enumType.getTypeName());
-        }
-    }
-
-    private void addRelationshipType(String relationshipTypeName, 
ExportContext context) {
-        if (!context.relationshipTypes.contains(relationshipTypeName)) {
-            AtlasRelationshipType relationshipType = 
typeRegistry.getRelationshipTypeByName(relationshipTypeName);
-
-            if (relationshipType != null) {
-                context.relationshipTypes.add(relationshipTypeName);
-
-                addAttributeTypes(relationshipType, context);
-                addEntityType(relationshipType.getEnd1Type(), context);
-                addEntityType(relationshipType.getEnd2Type(), context);
-            }
-        }
-    }
-
-    private void addAttributeTypes(AtlasStructType structType, ExportContext 
context) {
-        for (AtlasAttributeDef attributeDef : 
structType.getStructDef().getAttributeDefs()) {
-            addType(attributeDef.getTypeName(), context);
-        }
-    }
-
-    private void addRelationshipTypes(AtlasEntityType entityType, 
ExportContext context) {
-        for (Map.Entry<String, Map<String, AtlasAttribute>> entry : 
entityType.getRelationshipAttributes().entrySet()) {
-            for (String relationshipType : entry.getValue().keySet()) {
-                addRelationshipType(relationshipType, context);
-            }
-        }
-    }
-
     private List<Map<String, Object>> executeGremlinQuery(String query, 
ExportContext context) {
         try {
             return (List<Map<String, Object>>) 
atlasGraph.executeGremlinScript(context.scriptEngine, context.bindings, query, 
false);
@@ -740,6 +593,7 @@ public class ExportService {
         }
 
         public void addToSink(AtlasEntityWithExtInfo entityWithExtInfo) throws 
AtlasBaseException {
+            addToEntityCreationOrder(entityWithExtInfo.getEntity().getGuid());
             sink.add(entityWithExtInfo);
         }
 
@@ -750,9 +604,5 @@ public class ExportService {
         public void addToEntityCreationOrder(String guid) {
             entityCreationOrder.add(guid);
         }
-
-        public void addToEntityCreationOrder(Collection<String> guids) {
-            entityCreationOrder.addAll(guids);
-        }
     }
 }
diff --git 
a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportTypeProcessor.java
 
b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportTypeProcessor.java
index b513996..5bad615 100644
--- 
a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportTypeProcessor.java
+++ 
b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportTypeProcessor.java
@@ -42,11 +42,9 @@ class ExportTypeProcessor {
     private static final Logger LOG = 
LoggerFactory.getLogger(ExportTypeProcessor.class);
 
     private AtlasTypeRegistry typeRegistry;
-    private final ExportService.ExportContext context;
 
-    ExportTypeProcessor(AtlasTypeRegistry typeRegistry, 
ExportService.ExportContext context) {
+    ExportTypeProcessor(AtlasTypeRegistry typeRegistry) {
         this.typeRegistry = typeRegistry;
-        this.context = context;
     }
 
     public void addTypes(AtlasEntity entity, ExportService.ExportContext 
context) {
diff --git 
a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java 
b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
index bf23f81..1ce96a8 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
@@ -190,10 +190,10 @@ public class ZipSource implements EntityImportStream {
         T t;
         try {
             t = AtlasType.fromJson(jsonData, clazz);
-            if(t == null) {
-                throw new AtlasBaseException("Error converting file to JSON.");
+            if (t == null) {
+                LOG.error("Error converting file to JSON.");
+                return null;
             }
-
         } catch (Exception e) {
             throw new AtlasBaseException("Error converting file to JSON.", e);
         }
@@ -238,7 +238,7 @@ public class ZipSource implements EntityImportStream {
             currentPosition++;
             return getEntityWithExtInfo(this.iterator.next());
         } catch (AtlasBaseException e) {
-            LOG.error("getNextEntityWithExtInfo", e);
+            LOG.warn("getNextEntityWithExtInfo", e);
             return null;
         }
     }
diff --git 
a/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java
 
b/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java
index df9e280..e53f9a8 100644
--- 
a/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java
+++ 
b/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java
@@ -73,7 +73,7 @@ public class AdminExportImportTestIT extends BaseResourceIT {
 
     @Test(dependsOnMethods = "importData")
     public void exportData() throws AtlasServiceException, IOException, 
AtlasBaseException {
-        final int EXPECTED_CREATION_ORDER_SIZE = 10;
+        final int EXPECTED_CREATION_ORDER_SIZE = 6;
 
         AtlasExportRequest request = 
TestResourceFileUtils.readObjectFromJson(".", EXPORT_REQUEST_FILE, 
AtlasExportRequest.class);
         InputStream exportedStream = atlasClientV2.exportData(request);
@@ -81,7 +81,7 @@ public class AdminExportImportTestIT extends BaseResourceIT {
 
         ZipSource zs = new ZipSource(exportedStream);
         assertNotNull(zs.getExportResult());
-        assertTrue(zs.getCreationOrder().size() > 
EXPECTED_CREATION_ORDER_SIZE, "expected creationOrderSize > " + 
EXPECTED_CREATION_ORDER_SIZE + ", but found " + zs.getCreationOrder().size());
+        assertTrue(zs.getCreationOrder().size() >= 
EXPECTED_CREATION_ORDER_SIZE, "expected creationOrderSize > " + 
EXPECTED_CREATION_ORDER_SIZE + ", but found " + zs.getCreationOrder().size());
     }
 
     private void performImport(String fileToImport, int 
expectedProcessedEntitiesCount) throws AtlasServiceException {

Reply via email to