Repository: atlas Updated Branches: refs/heads/branch-0.8 dcce0f8d0 -> 44dd6a1a7
ATLAS-2804: Export & Import Detailed Audits. Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/44dd6a1a Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/44dd6a1a Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/44dd6a1a Branch: refs/heads/branch-0.8 Commit: 44dd6a1a7d33bbe8d428a2e9feba36a9b0ac3b79 Parents: dcce0f8 Author: Ashutosh Mestry <[email protected]> Authored: Thu Aug 2 13:31:59 2018 -0700 Committer: Ashutosh Mestry <[email protected]> Committed: Thu Aug 2 15:51:47 2018 -0700 ---------------------------------------------------------------------- addons/models/0010-base_model.json | 2 +- .../atlas/model/impexp/AtlasExportRequest.java | 19 ++-- .../atlas/model/impexp/AtlasExportResult.java | 17 ++- .../atlas/model/impexp/AtlasImportRequest.java | 1 + .../repository/clusterinfo/ClusterService.java | 2 + .../atlas/repository/impexp/AuditHelper.java | 113 +++++++++++++++++++ .../impexp/ExportImportAuditService.java | 2 + .../atlas/repository/impexp/ExportService.java | 19 +++- .../atlas/repository/impexp/ImportService.java | 16 ++- .../repository/impexp/ExportImportTestBase.java | 51 +++++++++ .../repository/impexp/ExportServiceTest.java | 26 ++++- .../repository/impexp/ImportServiceTest.java | 18 ++- .../atlas/web/resources/AdminResource.java | 36 +++++- .../atlas/web/resources/AdminResourceTest.java | 4 +- 14 files changed, 293 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/44dd6a1a/addons/models/0010-base_model.json ---------------------------------------------------------------------- diff --git a/addons/models/0010-base_model.json b/addons/models/0010-base_model.json index 59660c5..c6d251c 100644 --- a/addons/models/0010-base_model.json +++ b/addons/models/0010-base_model.json @@ -278,7 +278,7 @@ "typeName": "string", "cardinality": "SINGLE", "isIndexable": true, - "isOptional": false, + "isOptional": true, "isUnique": false }, { http://git-wip-us.apache.org/repos/asf/atlas/blob/44dd6a1a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportRequest.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportRequest.java b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportRequest.java index f1ed822..ce8aeb5 100644 --- a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportRequest.java +++ b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportRequest.java @@ -46,15 +46,16 @@ public class AtlasExportRequest implements Serializable { private static final long serialVersionUID = 1L; - public static final String OPTION_FETCH_TYPE = "fetchType"; - public static final String OPTION_ATTR_MATCH_TYPE = "matchType"; - public static final String FETCH_TYPE_FULL = "full"; - public static final String FETCH_TYPE_CONNECTED = "connected"; - public static final String MATCH_TYPE_STARTS_WITH = "startsWith"; - public static final String MATCH_TYPE_ENDS_WITH = "endsWith"; - public static final String MATCH_TYPE_CONTAINS = "contains"; - public static final String MATCH_TYPE_MATCHES = "matches"; - public static final String MATCH_TYPE_FOR_TYPE = "forType"; + public static final String OPTION_FETCH_TYPE = "fetchType"; + public static final String OPTION_ATTR_MATCH_TYPE = "matchType"; + public static final String OPTION_KEY_REPLICATED_TO = "replicatedTo"; + public static final String FETCH_TYPE_FULL = "full"; + public static final String FETCH_TYPE_CONNECTED = "connected"; + public static final String MATCH_TYPE_STARTS_WITH = "startsWith"; + public static final String MATCH_TYPE_ENDS_WITH = "endsWith"; + public static final String MATCH_TYPE_CONTAINS = "contains"; + public static final String MATCH_TYPE_MATCHES = "matches"; + public static final String MATCH_TYPE_FOR_TYPE = "forType"; private List<AtlasObjectId> itemsToExport = new ArrayList<>(); private Map<String, Object> options = new HashMap<>(); http://git-wip-us.apache.org/repos/asf/atlas/blob/44dd6a1a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportResult.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportResult.java b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportResult.java index 8f3075e..4da91a0 100644 --- a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportResult.java +++ b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportResult.java @@ -60,6 +60,7 @@ public class AtlasExportResult implements Serializable { private Map<String, Integer> metrics; private AtlasExportData data; private OperationStatus operationStatus; + private String sourceClusterName; public AtlasExportResult() { @@ -142,6 +143,13 @@ public class AtlasExportResult implements Serializable { this.operationStatus = operationStatus; } + public String getSourceClusterName() { + return sourceClusterName; + } + + public void setSourceClusterName(String sourceClusterName) { + this.sourceClusterName = sourceClusterName; + } public void incrementMeticsCounter(String key) { incrementMeticsCounter(key, 1); @@ -163,6 +171,7 @@ public class AtlasExportResult implements Serializable { sb.append(", userName='").append(userName).append("'"); sb.append(", clientIpAddress='").append(clientIpAddress).append("'"); sb.append(", hostName='").append(hostName).append("'"); + sb.append(", sourceCluster='").append(sourceClusterName).append("'"); sb.append(", timeStamp='").append(timeStamp).append("'"); sb.append(", metrics={"); AtlasBaseTypeDef.dumpObjects(metrics, sb); @@ -227,12 +236,12 @@ public class AtlasExportResult implements Serializable { sb = new StringBuilder(); } - sb.append("AtlasExportData{"); - sb.append("typesDef={").append(typesDef).append("}"); - sb.append("entities={"); + sb.append("AtlasExportData {"); + sb.append(", typesDef={").append(typesDef).append("}"); + sb.append(", entities={"); AtlasBaseTypeDef.dumpObjects(entities, sb); sb.append("}"); - sb.append("entityCreationOrder={"); + sb.append(", entityCreationOrder={"); AtlasBaseTypeDef.dumpObjects(entityCreationOrder, sb); sb.append("}"); sb.append("}"); http://git-wip-us.apache.org/repos/asf/atlas/blob/44dd6a1a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java index 304ec5d..f56fe22 100644 --- a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java +++ b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java @@ -38,6 +38,7 @@ public class AtlasImportRequest implements Serializable { private static final long serialVersionUID = 1L; public static final String TRANSFORMS_KEY = "transforms"; + public static final String OPTION_KEY_REPLICATED_FROM = "replicatedFrom"; private static final String START_POSITION_KEY = "startPosition"; private static final String START_GUID_KEY = "startGuid"; private static final String FILE_NAME_KEY = "fileName"; http://git-wip-us.apache.org/repos/asf/atlas/blob/44dd6a1a/repository/src/main/java/org/apache/atlas/repository/clusterinfo/ClusterService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/clusterinfo/ClusterService.java b/repository/src/main/java/org/apache/atlas/repository/clusterinfo/ClusterService.java index 0e23374..9c0afc0 100644 --- a/repository/src/main/java/org/apache/atlas/repository/clusterinfo/ClusterService.java +++ b/repository/src/main/java/org/apache/atlas/repository/clusterinfo/ClusterService.java @@ -19,6 +19,7 @@ package org.apache.atlas.repository.clusterinfo; import org.apache.atlas.annotation.AtlasService; +import org.apache.atlas.annotation.GraphTransaction; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.clusterinfo.AtlasCluster; import org.apache.atlas.repository.ogm.DataAccess; @@ -48,6 +49,7 @@ public class ClusterService { return null; } + @GraphTransaction public AtlasCluster save(AtlasCluster clusterInfo) throws AtlasBaseException { return dataAccess.save(clusterInfo); } http://git-wip-us.apache.org/repos/asf/atlas/blob/44dd6a1a/repository/src/main/java/org/apache/atlas/repository/impexp/AuditHelper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/AuditHelper.java b/repository/src/main/java/org/apache/atlas/repository/impexp/AuditHelper.java new file mode 100644 index 0000000..3137537 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/impexp/AuditHelper.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.impexp; + +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasConstants; +import org.apache.atlas.AtlasException; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.clusterinfo.AtlasCluster; +import org.apache.atlas.model.impexp.AtlasExportRequest; +import org.apache.atlas.model.impexp.AtlasExportResult; +import org.apache.atlas.model.impexp.AtlasImportRequest; +import org.apache.atlas.model.impexp.AtlasImportResult; +import org.apache.atlas.model.impexp.ExportImportAuditEntry; +import org.apache.atlas.repository.clusterinfo.ClusterService; +import org.apache.atlas.type.AtlasType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.util.Map; + +@Component +public class AuditHelper { + private static final Logger LOG = LoggerFactory.getLogger(AuditHelper.class); + private static final String CLUSTER_NAME_DEFAULT = "default"; + + private ClusterService clusterService; + private ExportImportAuditService auditService; + + @Inject + public AuditHelper(ClusterService clusterService, ExportImportAuditService auditService) { + this.clusterService = clusterService; + this.auditService = auditService; + } + + + public void audit(String userName, AtlasExportResult result, long startTime, long endTime, boolean hadData) throws AtlasBaseException { + AtlasExportRequest request = result.getRequest(); + AtlasCluster cluster = saveCluster(getCurrentClusterName()); + String targetClusterName = getClusterNameFromOptions(request.getOptions(), AtlasExportRequest.OPTION_KEY_REPLICATED_TO); + addAuditEntry(userName, + cluster.getName(), targetClusterName, + ExportImportAuditEntry.OPERATION_EXPORT, + AtlasType.toJson(result), startTime, endTime, hadData); + } + + public void audit(String userName, AtlasImportResult result, long startTime, long endTime, boolean hadData) throws AtlasBaseException { + AtlasImportRequest request = result.getRequest(); + AtlasCluster cluster = saveCluster(getCurrentClusterName()); + String sourceCluster = getClusterNameFromOptions(request.getOptions(), AtlasImportRequest.OPTION_KEY_REPLICATED_FROM); + addAuditEntry(userName, + sourceCluster, cluster.getName(), + ExportImportAuditEntry.OPERATION_EXPORT, AtlasType.toJson(result), startTime, endTime, hadData); + } + + private String getClusterNameFromOptions(Map options, String key) { + return options.containsKey(key) + ? (String) options.get(key) + : ""; + } + + private void addAuditEntry(String userName, String sourceCluster, String targetCluster, String operation, + String result, long startTime, long endTime, boolean hasData) throws AtlasBaseException { + if(!hasData) return; + + ExportImportAuditEntry entry = new ExportImportAuditEntry(); + + entry.setUserName(userName); + entry.setSourceClusterName(sourceCluster); + entry.setTargetClusterName(targetCluster); + entry.setOperation(operation); + entry.setResultSummary(result); + entry.setStartTime(startTime); + entry.setEndTime(endTime); + + auditService.save(entry); + LOG.info("addAuditEntry: user: {}, source: {}, target: {}, operation: {}", entry.getUserName(), + entry.getSourceClusterName(), entry.getTargetClusterName(), entry.getOperation()); + } + + private AtlasCluster saveCluster(String clusterName) throws AtlasBaseException { + AtlasCluster cluster = new AtlasCluster(clusterName, clusterName); + return clusterService.save(cluster); + } + + public static String getCurrentClusterName() { + try { + return ApplicationProperties.get().getString(AtlasConstants.CLUSTER_NAME_KEY, CLUSTER_NAME_DEFAULT); + } catch (AtlasException e) { + LOG.error("getCurrentClusterName", e); + } + + return ""; + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/44dd6a1a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportImportAuditService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportImportAuditService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportImportAuditService.java index ebfc33f..f7e32dc 100644 --- a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportImportAuditService.java +++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportImportAuditService.java @@ -19,6 +19,7 @@ package org.apache.atlas.repository.impexp; import org.apache.atlas.annotation.AtlasService; +import org.apache.atlas.annotation.GraphTransaction; import org.apache.atlas.discovery.AtlasDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.discovery.AtlasSearchResult; @@ -47,6 +48,7 @@ public class ExportImportAuditService { this.discoveryService = discoveryService; } + @GraphTransaction public void save(ExportImportAuditEntry entry) throws AtlasBaseException { dataAccess.saveNoLoad(entry); } http://git-wip-us.apache.org/repos/asf/atlas/blob/44dd6a1a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java index ce792e0..069f5c0 100644 --- a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java +++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java @@ -69,16 +69,18 @@ public class ExportService { private static final Logger LOG = LoggerFactory.getLogger(ExportService.class); private final AtlasTypeRegistry typeRegistry; + private AuditHelper auditHelper; private final AtlasGraph atlasGraph; private final EntityGraphRetriever entityGraphRetriever; private final AtlasGremlinQueryProvider gremlinQueryProvider; @Inject - public ExportService(final AtlasTypeRegistry typeRegistry, AtlasGraph atlasGraph) throws AtlasBaseException { + public ExportService(final AtlasTypeRegistry typeRegistry, AtlasGraph atlasGraph, AuditHelper auditHelper) { this.typeRegistry = typeRegistry; this.entityGraphRetriever = new EntityGraphRetriever(this.typeRegistry); this.atlasGraph = atlasGraph; this.gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE; + this.auditHelper = auditHelper; } public AtlasExportResult run(ZipSink exportSink, AtlasExportRequest request, String userName, String hostName, @@ -93,7 +95,8 @@ public class ExportService { AtlasExportResult.OperationStatus[] statuses = processItems(request, context); processTypesDef(context); - updateSinkWithOperationMetrics(context, statuses, getOperationDuration(startTime)); + long endTime = System.currentTimeMillis(); + updateSinkWithOperationMetrics(userName, context, statuses, startTime, endTime); } catch(Exception ex) { LOG.error("Operation failed: ", ex); } finally { @@ -106,10 +109,16 @@ public class ExportService { return context.result; } - private void updateSinkWithOperationMetrics(ExportContext context, AtlasExportResult.OperationStatus[] statuses, int duration) throws AtlasBaseException { + private void updateSinkWithOperationMetrics(String userName, ExportContext context, + AtlasExportResult.OperationStatus[] statuses, + long startTime, long endTime) throws AtlasBaseException { + int duration = getOperationDuration(startTime, endTime); + context.result.setSourceClusterName(AuditHelper.getCurrentClusterName()); context.result.getData().getEntityCreationOrder().addAll(context.lineageProcessed); context.sink.setExportOrder(context.result.getData().getEntityCreationOrder()); context.sink.setTypesDef(context.result.getData().getTypesDef()); + auditHelper.audit(userName, context.result, startTime, endTime, + !context.result.getData().getEntityCreationOrder().isEmpty()); clearContextData(context); context.result.setOperationStatus(getOverallOperationStatus(statuses)); context.result.incrementMeticsCounter("duration", duration); @@ -120,8 +129,8 @@ public class ExportService { context.result.setData(null); } - private int getOperationDuration(long startTime) { - return (int) (System.currentTimeMillis() - startTime); + private int getOperationDuration(long startTime, long endTime) { + return (int) (endTime - startTime); } private void processTypesDef(ExportContext context) { http://git-wip-us.apache.org/repos/asf/atlas/blob/44dd6a1a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java index 972e1e3..035b669 100644 --- a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java +++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java @@ -47,15 +47,17 @@ public class ImportService { private final AtlasTypeDefStore typeDefStore; private final AtlasTypeRegistry typeRegistry; private final BulkImporter bulkImporter; + private AuditHelper auditHelper; private long startTimestamp; private long endTimestamp; @Inject - public ImportService(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, BulkImporter bulkImporter) { + public ImportService(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, BulkImporter bulkImporter, AuditHelper auditHelper) { this.typeDefStore = typeDefStore; this.typeRegistry = typeRegistry; this.bulkImporter = bulkImporter; + this.auditHelper = auditHelper; } public AtlasImportResult run(ZipSource source, String userName, @@ -81,8 +83,7 @@ public class ImportService { startTimestamp = System.currentTimeMillis(); processTypes(source.getTypesDef(), result); setStartPosition(request, source); - processEntities(source, result); - + processEntities(userName, source, result); result.setOperationStatus(AtlasImportResult.OperationStatus.SUCCESS); } catch (AtlasBaseException excp) { @@ -183,10 +184,15 @@ public class ImportService { importTypeDefProcessor.processTypes(typeDefinitionMap, result); } - private void processEntities(ZipSource importSource, AtlasImportResult result) throws AtlasBaseException { + private void processEntities(String userName, ZipSource importSource, AtlasImportResult result) throws AtlasBaseException { this.bulkImporter.bulkImport(importSource, result); endTimestamp = System.currentTimeMillis(); - result.incrementMeticsCounter("duration", (int) (this.endTimestamp - this.startTimestamp)); + result.incrementMeticsCounter("duration", getDuration(this.endTimestamp, this.startTimestamp)); + auditHelper.audit(userName, result, startTimestamp, endTimestamp, !importSource.getCreationOrder().isEmpty()); + } + + private int getDuration(long endTime, long startTime) { + return (int) (endTime - startTime); } } http://git-wip-us.apache.org/repos/asf/atlas/blob/44dd6a1a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportTestBase.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportTestBase.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportTestBase.java new file mode 100644 index 0000000..bd4d4c9 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportTestBase.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.impexp; + +import com.google.inject.Inject; +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasConstants; +import org.apache.atlas.AtlasException; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.discovery.AtlasSearchResult; + +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +public class ExportImportTestBase { + + protected void assertAuditEntry(ExportImportAuditService auditService) { + AtlasSearchResult result = null; + try { + result = auditService.get("", "", getCurrentCluster(), "", "", "", 10, 0); + } catch (AtlasBaseException e) { + fail("auditService.get: failed!"); + } catch (AtlasException e) { + fail("getCurrentCluster: failed!"); + } + assertNotNull(result); + assertNotNull(result.getEntities()); + assertTrue(result.getEntities().size() > 0); + } + + private String getCurrentCluster() throws AtlasException { + return ApplicationProperties.get().getString(AtlasConstants.CLUSTER_NAME_KEY, "default"); + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/44dd6a1a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java index 8b68bba..a4026f6 100644 --- a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java @@ -18,12 +18,18 @@ package org.apache.atlas.repository.impexp; +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasConstants; +import org.apache.atlas.AtlasException; import org.apache.atlas.RequestContextV1; import org.apache.atlas.TestModules; import org.apache.atlas.TestUtilsV2; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.TypeCategory; +import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.impexp.AtlasExportRequest; import org.apache.atlas.model.impexp.AtlasExportResult; +import org.apache.atlas.model.impexp.ExportImportAuditEntry; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.typedef.AtlasTypesDef; @@ -36,12 +42,15 @@ import org.apache.atlas.repository.store.graph.v1.DeleteHandlerV1; import org.apache.atlas.repository.store.graph.v1.EntityGraphMapper; import org.apache.atlas.repository.store.graph.v1.SoftDeleteHandlerV1; import org.apache.atlas.store.AtlasTypeDefStore; +import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.AfterTest; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeTest; import org.testng.annotations.Guice; @@ -62,9 +71,10 @@ import static org.mockito.Mockito.mock; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; @Guice(modules = TestModules.TestOnlyModule.class) -public class ExportServiceTest { +public class ExportServiceTest extends ExportImportTestBase { private static final Logger LOG = LoggerFactory.getLogger(ExportServiceTest.class); @Inject @@ -75,16 +85,22 @@ public class ExportServiceTest { @Inject private EntityGraphMapper graphMapper; + @Inject ExportService exportService; + + @Inject + private ExportImportAuditService auditService; + private DeleteHandlerV1 deleteHandler = mock(SoftDeleteHandlerV1.class);; private AtlasEntityChangeNotifier mockChangeNotifier = mock(AtlasEntityChangeNotifier.class); private AtlasEntityStoreV1 entityStore; @BeforeTest - public void setupTest() { + public void setupTest() throws IOException, AtlasBaseException { RequestContextV1.clear(); RequestContextV1.get().setUser(TestUtilsV2.TEST_USER); + ZipFileResourceTestUtils.loadBaseModel(typeDefStore, typeRegistry); } @BeforeClass @@ -106,7 +122,9 @@ public class ExportServiceTest { } @AfterClass - public void clear() { + public void clear() throws InterruptedException { + Thread.sleep(1000); + assertAuditEntry(auditService); AtlasGraphProvider.cleanup(); } @@ -197,6 +215,7 @@ public class ExportServiceTest { assertEquals(result.getHostName(), hostName); assertEquals(result.getClientIpAddress(), requestingIP); assertEquals(request, result.getRequest()); + assertNotNull(result.getSourceClusterName()); } @Test @@ -392,4 +411,5 @@ public class ExportServiceTest { assertEquals(zipSource.getTypesDef().getStructDefs().size(), 1); assertEquals(zipSource.getTypesDef().getEntityDefs().size(), 4); } + } http://git-wip-us.apache.org/repos/asf/atlas/blob/44dd6a1a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java index c03fbc6..dc25e92 100644 --- a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java @@ -18,11 +18,15 @@ package org.apache.atlas.repository.impexp; import com.google.inject.Inject; +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasConstants; import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.AtlasException; import org.apache.atlas.RequestContextV1; import org.apache.atlas.TestModules; import org.apache.atlas.TestUtilsV2; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.impexp.AtlasImportRequest; import org.apache.atlas.store.AtlasTypeDefStore; import org.apache.atlas.type.AtlasClassificationType; @@ -32,6 +36,7 @@ import org.mockito.stubbing.Answer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.ITestContext; +import org.testng.annotations.AfterTest; import org.testng.annotations.BeforeTest; import org.testng.annotations.DataProvider; import org.testng.annotations.Guice; @@ -47,9 +52,10 @@ import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; @Guice(modules = TestModules.TestOnlyModule.class) -public class ImportServiceTest { +public class ImportServiceTest extends ExportImportTestBase { private static final Logger LOG = LoggerFactory.getLogger(ImportServiceTest.class); private final ImportService importService; @@ -60,6 +66,9 @@ public class ImportServiceTest { private AtlasTypeDefStore typeDefStore; @Inject + private ExportImportAuditService auditService; + + @Inject public ImportServiceTest(ImportService importService) { this.importService = importService; } @@ -70,6 +79,11 @@ public class ImportServiceTest { RequestContextV1.get().setUser(TestUtilsV2.TEST_USER); } + @AfterTest + public void postTest() { + assertAuditEntry(auditService); + } + @DataProvider(name = "sales") public static Object[][] getDataFromQuickStart_v1_Sales(ITestContext context) throws IOException { return getZipSource("sales-v1-full.zip"); @@ -190,7 +204,7 @@ public class ImportServiceTest { @Test public void importServiceProcessesIOException() { - ImportService importService = new ImportService(typeDefStore, typeRegistry, null); + ImportService importService = new ImportService(typeDefStore, typeRegistry, null, null); AtlasImportRequest req = mock(AtlasImportRequest.class); Answer<Map> answer = new Answer<Map>() { http://git-wip-us.apache.org/repos/asf/atlas/blob/44dd6a1a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java index 4224bb1..e9477ff 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java @@ -28,17 +28,19 @@ import org.apache.atlas.authorize.AtlasResourceTypes; import org.apache.atlas.authorize.simple.AtlasAuthorizationUtils; import org.apache.atlas.discovery.SearchContext; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.clusterinfo.AtlasCluster; import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.impexp.AtlasExportRequest; import org.apache.atlas.model.impexp.AtlasExportResult; import org.apache.atlas.model.impexp.AtlasImportRequest; import org.apache.atlas.model.impexp.AtlasImportResult; import org.apache.atlas.model.metrics.AtlasMetrics; +import org.apache.atlas.repository.clusterinfo.ClusterService; +import org.apache.atlas.repository.impexp.ExportImportAuditService; import org.apache.atlas.repository.impexp.ExportService; import org.apache.atlas.repository.impexp.ImportService; import org.apache.atlas.repository.impexp.ZipSink; import org.apache.atlas.repository.impexp.ZipSource; -import org.apache.atlas.repository.impexp.ExportImportAuditService; import org.apache.atlas.services.MetricsService; import org.apache.atlas.type.AtlasType; import org.apache.atlas.util.SearchTracker; @@ -119,6 +121,7 @@ public class AdminResource { private final ExportService exportService; private final ImportService importService; private final SearchTracker activeSearches; + private ClusterService clusterService; private ExportImportAuditService exportImportAuditService; static { @@ -132,14 +135,16 @@ public class AdminResource { @Inject public AdminResource(ServiceState serviceState, MetricsService metricsService, ExportService exportService, ImportService importService, SearchTracker activeSearches, + ClusterService clusterService, ExportImportAuditService exportImportAuditService) { this.serviceState = serviceState; this.metricsService = metricsService; this.exportService = exportService; this.importService = importService; this.activeSearches = activeSearches; + this.clusterService = clusterService; this.exportImportAuditService = exportImportAuditService; - importExportOperationLock = new ReentrantLock(); + this.importExportOperationLock = new ReentrantLock(); } /** @@ -431,6 +436,33 @@ public class AdminResource { return result; } + /** + * Fetch details of a cluster. + * @param clusterName name of target cluster with which it is paired + * @param entityQualifiedName qualified name of top level entity + * @return AtlasCluster + * @throws AtlasBaseException + */ + @GET + @Path("/cluster/{clusterName}") + @Consumes(Servlets.JSON_MEDIA_TYPE) + @Produces(Servlets.JSON_MEDIA_TYPE) + public AtlasCluster getCluster(@PathParam("clusterName") String clusterName, + @QueryParam("entity") String entityQualifiedName) throws AtlasBaseException { + AtlasPerfTracer perf = null; + + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "cluster.getCluster(" + clusterName + ")"); + } + + AtlasCluster cluster = new AtlasCluster(clusterName, clusterName); + return clusterService.get(cluster); + } finally { + AtlasPerfTracer.log(perf); + } + } + @GET @Path("/expimp/audit") @Consumes(Servlets.JSON_MEDIA_TYPE) http://git-wip-us.apache.org/repos/asf/atlas/blob/44dd6a1a/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java b/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java index 1fe3119..2f4b3d9 100644 --- a/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java +++ b/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java @@ -48,7 +48,7 @@ public class AdminResourceTest { when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.ACTIVE); - AdminResource adminResource = new AdminResource(serviceState, null, null, null, null, null); + AdminResource adminResource = new AdminResource(serviceState, null, null, null, null, null, null); Response response = adminResource.getStatus(); assertEquals(response.getStatus(), HttpServletResponse.SC_OK); JSONObject entity = (JSONObject) response.getEntity(); @@ -59,7 +59,7 @@ public class AdminResourceTest { public void testResourceGetsValueFromServiceState() throws JSONException { when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE); - AdminResource adminResource = new AdminResource(serviceState, null, null, null, null, null); + AdminResource adminResource = new AdminResource(serviceState, null, null, null, null, null, null); Response response = adminResource.getStatus(); verify(serviceState).getState();
