ATLAS-2806: Using replication attributes during export and import process.
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/c3b01a6f Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/c3b01a6f Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/c3b01a6f Branch: refs/heads/branch-0.8 Commit: c3b01a6f1e0b8c640c865c82ed22e2e4d66ae190 Parents: a1b6ba3 Author: Ashutosh Mestry <[email protected]> Authored: Mon Aug 6 12:02:39 2018 -0700 Committer: Ashutosh Mestry <[email protected]> Committed: Mon Aug 6 12:02:39 2018 -0700 ---------------------------------------------------------------------- .../004-base_model_replication_attributes.json | 29 + .../org/apache/atlas/repository/Constants.java | 8 + docs/src/site/twiki/Export-API.twiki | 4 +- pom.xml | 2 +- .../repository/clusterinfo/ClusterService.java | 56 -- .../atlas/repository/impexp/AuditHelper.java | 113 --- .../atlas/repository/impexp/AuditsWriter.java | 185 +++++ .../atlas/repository/impexp/ClusterService.java | 124 ++++ .../atlas/repository/impexp/ExportService.java | 11 +- .../atlas/repository/impexp/ImportService.java | 8 +- .../store/graph/v1/EntityGraphRetriever.java | 1 + .../clusterinfo/ClusterServiceTest.java | 1 + .../repository/impexp/ExportImportTestBase.java | 5 + .../impexp/ReplicationEntityAttributeTest.java | 205 ++++++ .../impexp/ZipFileResourceTestUtils.java | 20 +- .../resources/json/stocksDB-Entities/db.json | 24 + .../stocksDB-Entities/export-replicatedTo.json | 11 + .../stocksDB-Entities/export-skip-lineage.json | 11 + .../import-replicatedFrom.json | 5 + .../stocksDB-Entities/replicationAttrs.json | 8 + .../json/stocksDB-Entities/table-columns.json | 284 ++++++++ .../stocksDB-Entities/table-table-lineage.json | 44 ++ .../json/stocksDB-Entities/table-view.json | 107 +++ .../typesdef-new-classification.json | 22 + .../json/stocksDB-Entities/typesdef.json | 685 +++++++++++++++++++ .../java/org/apache/atlas/RequestContextV1.java | 2 +- .../atlas/web/resources/AdminResource.java | 2 +- 27 files changed, 1791 insertions(+), 186 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/addons/models/patches/004-base_model_replication_attributes.json ---------------------------------------------------------------------- diff --git a/addons/models/patches/004-base_model_replication_attributes.json b/addons/models/patches/004-base_model_replication_attributes.json new file mode 100644 index 0000000..bee3718 --- /dev/null +++ b/addons/models/patches/004-base_model_replication_attributes.json @@ -0,0 +1,29 @@ +{ + "patches": [ + { + "action": "ADD_ATTRIBUTE", + "typeName": "Referenceable", + "applyToVersion": "1.0", + "updateToVersion": "1.1", + "params": null, + "attributeDefs": [ + { + "name": "replicatedFromCluster", + "typeName": "array<AtlasCluster>", + "cardinality": "SET", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "replicatedToCluster", + "typeName": "array<AtlasCluster>", + "cardinality": "SET", + "isIndexable": false, + "isOptional": true, + "isUnique": false + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/common/src/main/java/org/apache/atlas/repository/Constants.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 4e179fb..56f341d 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -99,6 +99,14 @@ public final class Constants { public static final String MAX_FULLTEXT_QUERY_STR_LENGTH = "atlas.graph.fulltext-max-query-str-length"; public static final String MAX_DSL_QUERY_STR_LENGTH = "atlas.graph.dsl-max-query-str-length"; + /* + * replication attributes + */ + + public static final String ATTR_NAME_REFERENCEABLE = "Referenceable."; + public static final String ATTR_NAME_REPLICATED_TO_CLUSTER = "replicatedToCluster"; + public static final String ATTR_NAME_REPLICATED_FROM_CLUSTER = "replicatedFromCluster"; + private Constants() { } http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/docs/src/site/twiki/Export-API.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/Export-API.twiki b/docs/src/site/twiki/Export-API.twiki index 0df6629..2074999 100644 --- a/docs/src/site/twiki/Export-API.twiki +++ b/docs/src/site/twiki/Export-API.twiki @@ -145,6 +145,6 @@ curl -X POST -u adminuser:password -H "Content-Type: application/json" -H "Cache { "typeName": "DB", "uniqueAttributes": { "name": "Logging" } } ], - "options": "full" -}' "http://localhost:21000/api/atlas/admin/export"; > quickStartDB.zip + "options": { "full" } + }' "http://localhost:21000/api/atlas/admin/export"; > quickStartDB.zip </verbatim> http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 3fe0b78..40d483b 100644 --- a/pom.xml +++ b/pom.xml @@ -506,7 +506,7 @@ <slf4j.version>1.7.21</slf4j.version> <jetty.version>8.1.19.v20160209</jetty.version> <jetty.maven.plugin.version>9.2.12.v20150709</jetty.maven.plugin.version> - <jersey.version>1.19</jersey.version> + <jersey.version>1.19.3</jersey.version> <tinkerpop.version>2.6.0</tinkerpop.version> <titan.version>0.5.4</titan.version> <hadoop.version>2.7.1</hadoop.version> http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/main/java/org/apache/atlas/repository/clusterinfo/ClusterService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/clusterinfo/ClusterService.java b/repository/src/main/java/org/apache/atlas/repository/clusterinfo/ClusterService.java deleted file mode 100644 index 9c0afc0..0000000 --- a/repository/src/main/java/org/apache/atlas/repository/clusterinfo/ClusterService.java +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.repository.clusterinfo; - -import org.apache.atlas.annotation.AtlasService; -import org.apache.atlas.annotation.GraphTransaction; -import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.clusterinfo.AtlasCluster; -import org.apache.atlas.repository.ogm.DataAccess; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.inject.Inject; - -@AtlasService -public class ClusterService { - private static final Logger LOG = LoggerFactory.getLogger(ClusterService.class); - - private final DataAccess dataAccess; - - @Inject - public ClusterService(DataAccess dataAccess) { - this.dataAccess = dataAccess; - } - - public AtlasCluster get(AtlasCluster cluster) { - try { - return dataAccess.load(cluster); - } catch (AtlasBaseException e) { - LOG.error("dataAccess", e); - } - - return null; - } - - @GraphTransaction - public AtlasCluster save(AtlasCluster clusterInfo) throws AtlasBaseException { - return dataAccess.save(clusterInfo); - } -} http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/main/java/org/apache/atlas/repository/impexp/AuditHelper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/AuditHelper.java b/repository/src/main/java/org/apache/atlas/repository/impexp/AuditHelper.java deleted file mode 100644 index 3137537..0000000 --- a/repository/src/main/java/org/apache/atlas/repository/impexp/AuditHelper.java +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.repository.impexp; - -import org.apache.atlas.ApplicationProperties; -import org.apache.atlas.AtlasConstants; -import org.apache.atlas.AtlasException; -import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.clusterinfo.AtlasCluster; -import org.apache.atlas.model.impexp.AtlasExportRequest; -import org.apache.atlas.model.impexp.AtlasExportResult; -import org.apache.atlas.model.impexp.AtlasImportRequest; -import org.apache.atlas.model.impexp.AtlasImportResult; -import org.apache.atlas.model.impexp.ExportImportAuditEntry; -import org.apache.atlas.repository.clusterinfo.ClusterService; -import org.apache.atlas.type.AtlasType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; - -import javax.inject.Inject; -import java.util.Map; - -@Component -public class AuditHelper { - private static final Logger LOG = LoggerFactory.getLogger(AuditHelper.class); - private static final String CLUSTER_NAME_DEFAULT = "default"; - - private ClusterService clusterService; - private ExportImportAuditService auditService; - - @Inject - public AuditHelper(ClusterService clusterService, ExportImportAuditService auditService) { - this.clusterService = clusterService; - this.auditService = auditService; - } - - - public void audit(String userName, AtlasExportResult result, long startTime, long endTime, boolean hadData) throws AtlasBaseException { - AtlasExportRequest request = result.getRequest(); - AtlasCluster cluster = saveCluster(getCurrentClusterName()); - String targetClusterName = getClusterNameFromOptions(request.getOptions(), AtlasExportRequest.OPTION_KEY_REPLICATED_TO); - addAuditEntry(userName, - cluster.getName(), targetClusterName, - ExportImportAuditEntry.OPERATION_EXPORT, - AtlasType.toJson(result), startTime, endTime, hadData); - } - - public void audit(String userName, AtlasImportResult result, long startTime, long endTime, boolean hadData) throws AtlasBaseException { - AtlasImportRequest request = result.getRequest(); - AtlasCluster cluster = saveCluster(getCurrentClusterName()); - String sourceCluster = getClusterNameFromOptions(request.getOptions(), AtlasImportRequest.OPTION_KEY_REPLICATED_FROM); - addAuditEntry(userName, - sourceCluster, cluster.getName(), - ExportImportAuditEntry.OPERATION_EXPORT, AtlasType.toJson(result), startTime, endTime, hadData); - } - - private String getClusterNameFromOptions(Map options, String key) { - return options.containsKey(key) - ? (String) options.get(key) - : ""; - } - - private void addAuditEntry(String userName, String sourceCluster, String targetCluster, String operation, - String result, long startTime, long endTime, boolean hasData) throws AtlasBaseException { - if(!hasData) return; - - ExportImportAuditEntry entry = new ExportImportAuditEntry(); - - entry.setUserName(userName); - entry.setSourceClusterName(sourceCluster); - entry.setTargetClusterName(targetCluster); - entry.setOperation(operation); - entry.setResultSummary(result); - entry.setStartTime(startTime); - entry.setEndTime(endTime); - - auditService.save(entry); - LOG.info("addAuditEntry: user: {}, source: {}, target: {}, operation: {}", entry.getUserName(), - entry.getSourceClusterName(), entry.getTargetClusterName(), entry.getOperation()); - } - - private AtlasCluster saveCluster(String clusterName) throws AtlasBaseException { - AtlasCluster cluster = new AtlasCluster(clusterName, clusterName); - return clusterService.save(cluster); - } - - public static String getCurrentClusterName() { - try { - return ApplicationProperties.get().getString(AtlasConstants.CLUSTER_NAME_KEY, CLUSTER_NAME_DEFAULT); - } catch (AtlasException e) { - LOG.error("getCurrentClusterName", e); - } - - return ""; - } -} http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java b/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java new file mode 100644 index 0000000..6a3fbec --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java @@ -0,0 +1,185 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.impexp; + +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasConstants; +import org.apache.atlas.AtlasException; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.clusterinfo.AtlasCluster; +import org.apache.atlas.model.impexp.AtlasExportRequest; +import org.apache.atlas.model.impexp.AtlasExportResult; +import org.apache.atlas.model.impexp.AtlasImportRequest; +import org.apache.atlas.model.impexp.AtlasImportResult; +import org.apache.atlas.model.impexp.ExportImportAuditEntry; +import org.apache.atlas.repository.Constants; +import org.apache.atlas.type.AtlasType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.util.List; +import java.util.Map; + +@Component +public class AuditsWriter { + private static final Logger LOG = LoggerFactory.getLogger(AuditsWriter.class); + private static final String CLUSTER_NAME_DEFAULT = "default"; + + private ClusterService clusterService; + private ExportImportAuditService auditService; + + private ExportAudits auditForExport = new ExportAudits(); + private ImportAudits auditForImport = new ImportAudits(); + + @Inject + public AuditsWriter(ClusterService clusterService, ExportImportAuditService auditService) { + this.clusterService = clusterService; + this.auditService = auditService; + } + + public void write(String userName, AtlasExportResult result, long startTime, long endTime, List<String> entityCreationOrder) throws AtlasBaseException { + auditForExport.add(userName, result, startTime, endTime, entityCreationOrder); + } + + public void write(String userName, AtlasImportResult result, long startTime, long endTime, List<String> entityCreationOrder) throws AtlasBaseException { + auditForImport.add(userName, result, startTime, endTime, entityCreationOrder); + } + + private boolean isReplicationOptionSet(Map<String, ? extends Object> options, String replicatedKey) { + return options.containsKey(replicatedKey); + } + + private void updateReplicationAttribute(boolean isReplicationSet, String clusterName, + List<String> exportedGuids, + String attrNameReplicated) throws AtlasBaseException { + if (!isReplicationSet) { + return; + } + + AtlasCluster cluster = saveCluster(clusterName); + clusterService.updateEntityWithCluster(cluster, exportedGuids, attrNameReplicated); + } + + private String getClusterNameFromOptions(Map options, String key) { + return options.containsKey(key) + ? (String) options.get(key) + : ""; + } + + private void addAuditEntry(String userName, String sourceCluster, String targetCluster, String operation, + String result, long startTime, long endTime, boolean hasData) throws AtlasBaseException { + if(!hasData) return; + + ExportImportAuditEntry entry = new ExportImportAuditEntry(); + + entry.setUserName(userName); + entry.setSourceClusterName(sourceCluster); + entry.setTargetClusterName(targetCluster); + entry.setOperation(operation); + entry.setResultSummary(result); + entry.setStartTime(startTime); + entry.setEndTime(endTime); + + auditService.save(entry); + LOG.info("addAuditEntry: user: {}, source: {}, target: {}, operation: {}", entry.getUserName(), + entry.getSourceClusterName(), entry.getTargetClusterName(), entry.getOperation()); + } + + private AtlasCluster saveCluster(String clusterName) throws AtlasBaseException { + AtlasCluster cluster = new AtlasCluster(clusterName, clusterName); + return clusterService.save(cluster); + } + + public static String getCurrentClusterName() { + try { + return ApplicationProperties.get().getString(AtlasConstants.CLUSTER_NAME_KEY, CLUSTER_NAME_DEFAULT); + } catch (AtlasException e) { + LOG.error("getCurrentClusterName", e); + } + + return ""; + } + + private class ExportAudits { + private AtlasExportRequest request; + private AtlasCluster cluster; + private String targetClusterName; + private String optionKeyReplicatedTo; + private boolean replicationOptionState; + + public void add(String userName, AtlasExportResult result, long startTime, long endTime, List<String> entitityGuids) throws AtlasBaseException { + optionKeyReplicatedTo = AtlasExportRequest.OPTION_KEY_REPLICATED_TO; + request = result.getRequest(); + cluster = saveCluster(getCurrentClusterName()); + replicationOptionState = isReplicationOptionSet(request.getOptions(), optionKeyReplicatedTo); + targetClusterName = getClusterNameFromOptions(request.getOptions(), optionKeyReplicatedTo); + + addAuditEntry(userName, + cluster.getName(), targetClusterName, + ExportImportAuditEntry.OPERATION_EXPORT, + AtlasType.toJson(result), startTime, endTime, !entitityGuids.isEmpty()); + + updateReplicationAttributeForExport(entitityGuids, request); + } + + private void updateReplicationAttributeForExport(List<String> entityGuids, AtlasExportRequest request) throws AtlasBaseException { + if(!replicationOptionState) return; + + updateReplicationAttribute(replicationOptionState, targetClusterName, entityGuids, Constants.ATTR_NAME_REPLICATED_TO_CLUSTER); + } + } + + private class ImportAudits { + private AtlasImportRequest request; + private boolean replicationOptionState; + private AtlasCluster cluster; + private String optionKeyReplicatedFrom; + private AtlasImportResult result; + + public void add(String userName, AtlasImportResult result, long startTime, long endTime, List<String> entitityGuids) throws AtlasBaseException { + this.result = result; + request = result.getRequest(); + optionKeyReplicatedFrom = AtlasImportRequest.OPTION_KEY_REPLICATED_FROM; + replicationOptionState = isReplicationOptionSet(request.getOptions(), optionKeyReplicatedFrom); + cluster = saveCluster(getClusterNameFromOptionsState()); + + String sourceCluster = getClusterNameFromOptions(request.getOptions(), optionKeyReplicatedFrom); + addAuditEntry(userName, + sourceCluster, cluster.getName(), + ExportImportAuditEntry.OPERATION_EXPORT, AtlasType.toJson(result), startTime, endTime, !entitityGuids.isEmpty()); + + updateReplicationAttributeForImport(entitityGuids); + } + + private void updateReplicationAttributeForImport(List<String> entityGuids) throws AtlasBaseException { + if(!replicationOptionState) return; + + String targetClusterName = cluster.getName(); + updateReplicationAttribute(replicationOptionState, targetClusterName, entityGuids, Constants.ATTR_NAME_REPLICATED_FROM_CLUSTER); + } + + private String getClusterNameFromOptionsState() { + return replicationOptionState + ? getClusterNameFromOptions(request.getOptions(), optionKeyReplicatedFrom) + : getCurrentClusterName(); + } + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/main/java/org/apache/atlas/repository/impexp/ClusterService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ClusterService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ClusterService.java new file mode 100644 index 0000000..fd8e2bf --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ClusterService.java @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.impexp; + +import org.apache.atlas.RequestContextV1; +import org.apache.atlas.annotation.AtlasService; +import org.apache.atlas.annotation.GraphTransaction; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.clusterinfo.AtlasCluster; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasObjectId; +import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.graph.GraphHelper; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.ogm.DataAccess; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.repository.store.graph.v1.AtlasEntityStream; +import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1; +import org.apache.atlas.repository.store.graph.v1.EntityGraphMapper; +import org.apache.atlas.repository.store.graph.v1.EntityGraphRetriever; +import org.apache.atlas.type.AtlasType; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; + +@AtlasService +public class ClusterService { + private static final Logger LOG = LoggerFactory.getLogger(ClusterService.class); + + private final DataAccess dataAccess; + private final AtlasEntityStore entityStore; + + @Inject + public ClusterService(DataAccess dataAccess, AtlasEntityStore entityStore) { + this.dataAccess = dataAccess; + this.entityStore = entityStore; + } + + public AtlasCluster get(AtlasCluster cluster) { + try { + return dataAccess.load(cluster); + } catch (AtlasBaseException e) { + LOG.error("dataAccess", e); + } + + return null; + } + + @GraphTransaction + public AtlasCluster save(AtlasCluster clusterInfo) throws AtlasBaseException { + return dataAccess.save(clusterInfo); + } + + @GraphTransaction + public void updateEntityWithCluster(AtlasCluster cluster, List<String> guids, String attributeName) throws AtlasBaseException { + if(cluster != null && StringUtils.isEmpty(cluster.getGuid())) return; + + AtlasObjectId objectId = getObjectId(cluster); + for (String guid : guids) { + AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = entityStore.getById(guid); + updateAttribute(entityWithExtInfo, attributeName, objectId); + entityStore.createOrUpdate(new AtlasEntityStream(entityWithExtInfo), true); + } + } + + private AtlasObjectId getObjectId(AtlasCluster cluster) { + return new AtlasObjectId(cluster.getGuid(), AtlasCluster.class.getSimpleName()); + } + + + /** + * Attribute passed by name is updated with the value passed. + * @param entityWithExtInfo Entity to be updated + * @param propertyName attribute name + * @param value Value to be set for attribute + */ + private void updateAttribute(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo, String propertyName, Object value) { + updateAttribute(entityWithExtInfo.getEntity(), propertyName, value); + for (AtlasEntity e : entityWithExtInfo.getReferredEntities().values()) { + updateAttribute(e, propertyName, value); + } + } + + private void updateAttribute(AtlasEntity e, String propertyName, Object value) { + if(e.hasAttribute(propertyName) == false) return; + + Object oVal = e.getAttribute(propertyName); + if (oVal != null && !(oVal instanceof List)) return; + + List list; + + if (oVal == null) { + list = new ArrayList(); + } else { + list = (List) oVal; + } + + if (!list.contains(value)) { + list.add(value); + } + + e.setAttribute(propertyName, list); + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java index 069f5c0..02d17fb 100644 --- a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java +++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java @@ -69,18 +69,18 @@ public class ExportService { private static final Logger LOG = LoggerFactory.getLogger(ExportService.class); private final AtlasTypeRegistry typeRegistry; - private AuditHelper auditHelper; + private AuditsWriter auditsWriter; private final AtlasGraph atlasGraph; private final EntityGraphRetriever entityGraphRetriever; private final AtlasGremlinQueryProvider gremlinQueryProvider; @Inject - public ExportService(final AtlasTypeRegistry typeRegistry, AtlasGraph atlasGraph, AuditHelper auditHelper) { + public ExportService(final AtlasTypeRegistry typeRegistry, AtlasGraph atlasGraph, AuditsWriter auditsWriter) { this.typeRegistry = typeRegistry; this.entityGraphRetriever = new EntityGraphRetriever(this.typeRegistry); this.atlasGraph = atlasGraph; this.gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE; - this.auditHelper = auditHelper; + this.auditsWriter = auditsWriter; } public AtlasExportResult run(ZipSink exportSink, AtlasExportRequest request, String userName, String hostName, @@ -113,12 +113,11 @@ public class ExportService { AtlasExportResult.OperationStatus[] statuses, long startTime, long endTime) throws AtlasBaseException { int duration = getOperationDuration(startTime, endTime); - context.result.setSourceClusterName(AuditHelper.getCurrentClusterName()); + context.result.setSourceClusterName(AuditsWriter.getCurrentClusterName()); context.result.getData().getEntityCreationOrder().addAll(context.lineageProcessed); context.sink.setExportOrder(context.result.getData().getEntityCreationOrder()); context.sink.setTypesDef(context.result.getData().getTypesDef()); - auditHelper.audit(userName, context.result, startTime, endTime, - !context.result.getData().getEntityCreationOrder().isEmpty()); + auditsWriter.write(userName, context.result, startTime, endTime, context.result.getData().getEntityCreationOrder()); clearContextData(context); context.result.setOperationStatus(getOverallOperationStatus(statuses)); context.result.incrementMeticsCounter("duration", duration); http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java index 035b669..98ef389 100644 --- a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java +++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java @@ -47,17 +47,17 @@ public class ImportService { private final AtlasTypeDefStore typeDefStore; private final AtlasTypeRegistry typeRegistry; private final BulkImporter bulkImporter; - private AuditHelper auditHelper; + private AuditsWriter auditsWriter; private long startTimestamp; private long endTimestamp; @Inject - public ImportService(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, BulkImporter bulkImporter, AuditHelper auditHelper) { + public ImportService(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, BulkImporter bulkImporter, AuditsWriter auditsWriter) { this.typeDefStore = typeDefStore; this.typeRegistry = typeRegistry; this.bulkImporter = bulkImporter; - this.auditHelper = auditHelper; + this.auditsWriter = auditsWriter; } public AtlasImportResult run(ZipSource source, String userName, @@ -189,7 +189,7 @@ public class ImportService { endTimestamp = System.currentTimeMillis(); result.incrementMeticsCounter("duration", getDuration(this.endTimestamp, this.startTimestamp)); - auditHelper.audit(userName, result, startTimestamp, endTimestamp, !importSource.getCreationOrder().isEmpty()); + auditsWriter.write(userName, result, startTimestamp, endTimestamp, importSource.getCreationOrder()); } private int getDuration(long endTime, long startTime) { http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java index 6bb4554..b972946 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java @@ -18,6 +18,7 @@ package org.apache.atlas.repository.store.graph.v1; import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.annotation.GraphTransaction; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasClassification; import org.apache.atlas.model.instance.AtlasEntity; http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/java/org/apache/atlas/repository/clusterinfo/ClusterServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/clusterinfo/ClusterServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/clusterinfo/ClusterServiceTest.java index 65962cc..dba640f 100644 --- a/repository/src/test/java/org/apache/atlas/repository/clusterinfo/ClusterServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/clusterinfo/ClusterServiceTest.java @@ -21,6 +21,7 @@ package org.apache.atlas.repository.clusterinfo; import org.apache.atlas.TestModules; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.clusterinfo.AtlasCluster; +import org.apache.atlas.repository.impexp.ClusterService; import org.apache.atlas.store.AtlasTypeDefStore; import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportTestBase.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportTestBase.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportTestBase.java index bd4d4c9..41c8486 100644 --- a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportTestBase.java +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportTestBase.java @@ -24,13 +24,18 @@ import org.apache.atlas.AtlasConstants; import org.apache.atlas.AtlasException; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.discovery.AtlasSearchResult; +import org.apache.atlas.repository.store.graph.v1.DeleteHandlerV1; +import org.apache.atlas.repository.store.graph.v1.SoftDeleteHandlerV1; +import static org.mockito.Mockito.mock; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; public class ExportImportTestBase { + protected DeleteHandlerV1 deleteHandler = mock(SoftDeleteHandlerV1.class); + protected void assertAuditEntry(ExportImportAuditService auditService) { AtlasSearchResult result = null; try { http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java new file mode 100644 index 0000000..540950a --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java @@ -0,0 +1,205 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.impexp; + + +import com.google.common.collect.ImmutableList; +import org.apache.atlas.RequestContextV1; +import org.apache.atlas.TestModules; +import org.apache.atlas.TestUtilsV2; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.clusterinfo.AtlasCluster; +import org.apache.atlas.model.impexp.AtlasExportRequest; +import org.apache.atlas.model.impexp.AtlasImportRequest; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.store.graph.v1.AtlasEntityChangeNotifier; +import org.apache.atlas.repository.store.graph.v1.AtlasEntityStoreV1; +import org.apache.atlas.repository.store.graph.v1.EntityGraphMapper; +import org.apache.atlas.store.AtlasTypeDefStore; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.TestResourceFileUtils; +import org.testng.SkipException; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Guice; +import org.testng.annotations.Test; + +import javax.inject.Inject; +import java.io.IOException; +import java.util.List; + +import static org.apache.atlas.model.impexp.AtlasExportRequest.OPTION_KEY_REPLICATED_TO; +import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.createAtlasEntity; +import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadBaseModel; +import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadEntity; +import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadHiveModel; +import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runExportWithParameters; +import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runImportWithParameters; +import static org.mockito.Mockito.mock; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; + +@Guice(modules = TestModules.TestOnlyModule.class) +public class ReplicationEntityAttributeTest extends ExportImportTestBase { + private final String ENTITIES_SUB_DIR = "stocksDB-Entities"; + private final String EXPORT_REQUEST_FILE = "export-replicatedTo"; + private final String IMPORT_REQUEST_FILE = "import-replicatedFrom"; + + private final String DB_GUID = "1637a33e-6512-447b-ade7-249c8cb5344b"; + private final String TABLE_GUID = "df122fc3-5555-40f8-a30f-3090b8a622f8"; + + private String REPLICATED_TO_CLUSTER_NAME = ""; + private String REPLICATED_FROM_CLUSTER_NAME = ""; + + @Inject + AtlasTypeRegistry typeRegistry; + + @Inject + private AtlasTypeDefStore typeDefStore; + + @Inject + private EntityGraphMapper graphMapper; + + @Inject + ExportService exportService; + + @Inject + ImportService importService; + + @Inject + ClusterService clusterService; + + private AtlasEntityChangeNotifier mockChangeNotifier = mock(AtlasEntityChangeNotifier.class); + private AtlasEntityStoreV1 entityStore; + private ZipSource zipSource; + + @BeforeClass + public void setup() throws IOException, AtlasBaseException { + loadBaseModel(typeDefStore, typeRegistry); + loadHiveModel(typeDefStore, typeRegistry); + createEntities(); + } + + private void createEntities() { + entityStore = new AtlasEntityStoreV1(deleteHandler, typeRegistry, mockChangeNotifier, graphMapper); + + createAtlasEntity(entityStore, loadEntity(ENTITIES_SUB_DIR,"db")); + createAtlasEntity(entityStore, loadEntity(ENTITIES_SUB_DIR, "table-columns")); + + try { + AtlasEntity.AtlasEntitiesWithExtInfo entities = entityStore.getByIds(ImmutableList.of(DB_GUID, TABLE_GUID)); + assertEquals(entities.getEntities().size(), 2); + } catch (AtlasBaseException e) { + throw new SkipException(String.format("getByIds: could not load '%s' & '%s'.", DB_GUID, TABLE_GUID)); + } + } + + @BeforeMethod + public void setupTest() { + RequestContextV1.clear(); + RequestContextV1.get().setUser(TestUtilsV2.TEST_USER); + } + + @Test + public void exportWithReplicationToOption_AddsClusterObjectIdToReplicatedFromAttribute() throws AtlasBaseException { + final int expectedEntityCount = 2; + + AtlasExportRequest request = getUpdateMetaInfoUpdateRequest(); + zipSource = runExportWithParameters(exportService, request); + + assertNotNull(zipSource); + assertNotNull(zipSource.getCreationOrder()); + assertEquals(zipSource.getCreationOrder().size(), expectedEntityCount); + + assertClusterInfo(REPLICATED_TO_CLUSTER_NAME); + assertReplicationAttribute(Constants.ATTR_NAME_REPLICATED_TO_CLUSTER); + } + + @Test(dependsOnMethods = "exportWithReplicationToOption_AddsClusterObjectIdToReplicatedFromAttribute") + public void importWithReplicationFromOption_AddsClusterObjectIdToReplicatedFromAttribute() throws AtlasBaseException, IOException { + AtlasImportRequest request = getImportRequestWithReplicationOption(); + runImportWithParameters(importService, request, zipSource); + + assertClusterInfo(REPLICATED_FROM_CLUSTER_NAME); + assertReplicationAttribute(Constants.ATTR_NAME_REPLICATED_FROM_CLUSTER); + } + + private void assertReplicationAttribute(String attrNameReplication) throws AtlasBaseException { + AtlasEntity.AtlasEntitiesWithExtInfo entities = entityStore.getByIds(ImmutableList.of(DB_GUID, TABLE_GUID)); + for (AtlasEntity e : entities.getEntities()) { + Object ex = e.getAttribute(attrNameReplication); + assertNotNull(ex); + + List<String> clusterNameSyncType = (List) ex; + assertEquals(clusterNameSyncType.size(), 1); + } + } + + private void assertClusterInfo(String name) { + AtlasCluster actual = clusterService.get(new AtlasCluster(name, name)); + + assertNotNull(actual); + assertEquals(actual.getName(), name); + } + + private AtlasExportRequest getUpdateMetaInfoUpdateRequest() { + AtlasExportRequest request = getExportRequestWithReplicationOption(); + request.getOptions().put(AtlasExportRequest.OPTION_KEY_REPLICATED_TO, REPLICATED_TO_CLUSTER_NAME); + + return request; + } + + private AtlasEntity.AtlasEntityWithExtInfo getEntities(ZipSource source, int expectedCount) { + AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = new AtlasEntity.AtlasEntityWithExtInfo(); + try { + int count = 0; + for(String s : source.getCreationOrder()) { + AtlasEntity entity = source.getByGuid(s); + entityWithExtInfo.addReferredEntity(s, entity); + count++; + } + + assertEquals(count, expectedCount); + return entityWithExtInfo; + } catch (AtlasBaseException e) { + throw new SkipException("getEntities: failed!"); + } + } + + private AtlasExportRequest getExportRequestWithReplicationOption() { + try { + AtlasExportRequest request = TestResourceFileUtils.readObjectFromJson(ENTITIES_SUB_DIR, EXPORT_REQUEST_FILE, AtlasExportRequest.class); + REPLICATED_TO_CLUSTER_NAME = (String) request.getOptions().get(OPTION_KEY_REPLICATED_TO); + return request; + } catch (IOException e) { + throw new SkipException(String.format("getExportRequestWithReplicationOption: '%s' could not be laoded.", EXPORT_REQUEST_FILE)); + } + } + + private AtlasImportRequest getImportRequestWithReplicationOption() { + try { + AtlasImportRequest request = TestResourceFileUtils.readObjectFromJson(ENTITIES_SUB_DIR, IMPORT_REQUEST_FILE, AtlasImportRequest.class); + REPLICATED_FROM_CLUSTER_NAME = request.getOptions().get(AtlasImportRequest.OPTION_KEY_REPLICATED_FROM); + return request; + } catch (IOException e) { + throw new SkipException(String.format("getExportRequestWithReplicationOption: '%s' could not be laoded.", IMPORT_REQUEST_FILE)); + } + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java index 6f01822..014fe75 100644 --- a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java @@ -27,16 +27,19 @@ import org.apache.atlas.model.impexp.AtlasImportRequest; import org.apache.atlas.model.impexp.AtlasImportResult; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.model.typedef.AtlasEntityDef; +import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer; import org.apache.atlas.repository.store.graph.v1.AtlasEntityStoreV1; import org.apache.atlas.repository.store.graph.v1.AtlasEntityStreamForImport; import org.apache.atlas.store.AtlasTypeDefStore; +import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.TestResourceFileUtils; import org.apache.commons.io.FileUtils; -import org.apache.solr.common.StringUtils; +import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.SkipException; @@ -65,7 +68,8 @@ public class ZipFileResourceTestUtils { public static String getModelJson(String fileName) throws IOException { final String userDir = System.getProperty("user.dir"); - String filePath = userDir + "/../addons/models/" + fileName; + String modelsDir = "/../addons/models/"; + String filePath = userDir + modelsDir + fileName; File f = new File(filePath); String s = FileUtils.readFileToString(f); assertFalse(StringUtils.isEmpty(s), "Model file read correctly!"); @@ -190,9 +194,21 @@ public class ZipFileResourceTestUtils { public static void loadModelFromJson(String fileName, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws IOException, AtlasBaseException { AtlasTypesDef typesFromJson = getAtlasTypesDefFromFile(fileName); + addReplicationAttributes(typesFromJson); createTypesAsNeeded(typesFromJson, typeDefStore, typeRegistry); } + private static void addReplicationAttributes(AtlasTypesDef typesFromJson) throws IOException { + AtlasEntityDef ed = typesFromJson.getEntityDefs().get(0); + if(!ed.getName().equals("Referenceable")) return; + + String replAttr1Json = TestResourceFileUtils.getJson("stocksDB-Entities","replicationAttrs"); + String replAttr2Json = StringUtils.replace(replAttr1Json, "From", "To"); + + ed.addAttribute(AtlasType.fromJson(replAttr1Json, AtlasStructDef.AtlasAttributeDef.class)); + ed.addAttribute(AtlasType.fromJson(replAttr2Json, AtlasStructDef.AtlasAttributeDef.class)); + } + public static void loadModelFromResourcesJson(String fileName, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws IOException, AtlasBaseException { AtlasTypesDef typesFromJson = getAtlasTypesDefFromResourceFile(fileName); createTypesAsNeeded(typesFromJson, typeDefStore, typeRegistry); http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/resources/json/stocksDB-Entities/db.json ---------------------------------------------------------------------- diff --git a/repository/src/test/resources/json/stocksDB-Entities/db.json b/repository/src/test/resources/json/stocksDB-Entities/db.json new file mode 100644 index 0000000..db581d5 --- /dev/null +++ b/repository/src/test/resources/json/stocksDB-Entities/db.json @@ -0,0 +1,24 @@ +{ + "entity": { + "attributes": { + "clusterName": "cl1", + "description": null, + "location": "hdfs://localhost.localdomain:8020/apps/hive/warehouse/stocks_base.db", + "name": "stocks_base", + "owner": "anonymous", + "ownerType": "USER", + "parameters": null, + "qualifiedName": "stocks_base@cl1" + }, + "classifications": [], + "createTime": 1528238690000, + "createdBy": "anonymous", + "guid": "1637a33e-6512-447b-ade7-249c8cb5344b", + "status": "ACTIVE", + "typeName": "hive_db", + "updateTime": 1528238690000, + "updatedBy": "anonymous", + "version": 0 + }, + "referredEntities": {} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/resources/json/stocksDB-Entities/export-replicatedTo.json ---------------------------------------------------------------------- diff --git a/repository/src/test/resources/json/stocksDB-Entities/export-replicatedTo.json b/repository/src/test/resources/json/stocksDB-Entities/export-replicatedTo.json new file mode 100644 index 0000000..a69fe9e --- /dev/null +++ b/repository/src/test/resources/json/stocksDB-Entities/export-replicatedTo.json @@ -0,0 +1,11 @@ +{ + "itemsToExport": [ + { + "typeName": "hive_db", "uniqueAttributes": { "qualifiedName": "stocks_base@cl1" } + } + ], + "options": { + "fetchType": "full", + "replicatedTo": "clTarget" + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/resources/json/stocksDB-Entities/export-skip-lineage.json ---------------------------------------------------------------------- diff --git a/repository/src/test/resources/json/stocksDB-Entities/export-skip-lineage.json b/repository/src/test/resources/json/stocksDB-Entities/export-skip-lineage.json new file mode 100644 index 0000000..f14eaca --- /dev/null +++ b/repository/src/test/resources/json/stocksDB-Entities/export-skip-lineage.json @@ -0,0 +1,11 @@ +{ + "itemsToExport": [ + { + "typeName": "hive_db", "uniqueAttributes": { "qualifiedName": "stocks_base@cl1" } + } + ], + "options": { + "fetchType": "incremental", + "skipLineage": true + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/resources/json/stocksDB-Entities/import-replicatedFrom.json ---------------------------------------------------------------------- diff --git a/repository/src/test/resources/json/stocksDB-Entities/import-replicatedFrom.json b/repository/src/test/resources/json/stocksDB-Entities/import-replicatedFrom.json new file mode 100644 index 0000000..1ce00ad --- /dev/null +++ b/repository/src/test/resources/json/stocksDB-Entities/import-replicatedFrom.json @@ -0,0 +1,5 @@ +{ + "options": { + "replicatedFrom": "clSource" + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/resources/json/stocksDB-Entities/replicationAttrs.json ---------------------------------------------------------------------- diff --git a/repository/src/test/resources/json/stocksDB-Entities/replicationAttrs.json b/repository/src/test/resources/json/stocksDB-Entities/replicationAttrs.json new file mode 100644 index 0000000..8282638 --- /dev/null +++ b/repository/src/test/resources/json/stocksDB-Entities/replicationAttrs.json @@ -0,0 +1,8 @@ +{ + "name": "replicatedFromCluster", + "typeName": "array<AtlasCluster>", + "cardinality": "SET", + "isIndexable": false, + "isOptional": true, + "isUnique": false +} http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/resources/json/stocksDB-Entities/table-columns.json ---------------------------------------------------------------------- diff --git a/repository/src/test/resources/json/stocksDB-Entities/table-columns.json b/repository/src/test/resources/json/stocksDB-Entities/table-columns.json new file mode 100644 index 0000000..1536897 --- /dev/null +++ b/repository/src/test/resources/json/stocksDB-Entities/table-columns.json @@ -0,0 +1,284 @@ +{ + "entity": { + "attributes": { + "aliases": null, + "columns": [ + { + "guid": "08bc6cee-fe25-47d2-9e39-155d4b807765", + "typeName": "hive_column" + }, + { + "guid": "b3a13f06-4585-4fef-a5ae-970c24bd3981", + "typeName": "hive_column" + }, + { + "guid": "f87a5320-1529-4369-8d63-b637ebdf2c1c", + "typeName": "hive_column" + }, + { + "guid": "47d732d5-a6ef-41b8-83bd-a54365750a47", + "typeName": "hive_column" + }, + { + "guid": "92a6a9b8-dbab-4313-a6f6-0035246b0e37", + "typeName": "hive_column" + }, + { + "guid": "7193839e-10d3-4926-83d8-2c675a1068bd", + "typeName": "hive_column" + }, + { + "guid": "c6e27ba6-9f4d-4e07-9a54-0fc631529f51", + "typeName": "hive_column" + } + ], + "comment": null, + "createTime": 1528238679000, + "db": { + "guid": "1637a33e-6512-447b-ade7-249c8cb5344b", + "typeName": "hive_db" + }, + "description": null, + "lastAccessTime": 1528238679000, + "name": "stocks_daily", + "owner": "anonymous", + "parameters": { + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "numFiles": "0", + "numRows": "0", + "rawDataSize": "0", + "totalSize": "0", + "transient_lastDdlTime": "1528238679" + }, + "partitionKeys": null, + "qualifiedName": "stocks_base.stocks_daily@cl1", + "retention": 0, + "sd": { + "guid": "1ab9f789-b56c-4b01-a6e4-523f0fe5cdbe", + "typeName": "hive_storagedesc" + }, + "tableType": "MANAGED_TABLE", + "temporary": false, + "viewExpandedText": null, + "viewOriginalText": null + }, + "classifications": [], + "createTime": 1528238679952, + "createdBy": "anonymous", + "guid": "df122fc3-5555-40f8-a30f-3090b8a622f8", + "status": "ACTIVE", + "typeName": "hive_table", + "updateTime": 1528238690000, + "updatedBy": "anonymous", + "version": 0 + }, + "referredEntities": { + "08bc6cee-fe25-47d2-9e39-155d4b807765": { + "attributes": { + "comment": null, + "description": null, + "name": "dt", + "owner": "anonymous", + "position": 0, + "qualifiedName": "stocks_base.stocks_daily.dt@cl1", + "table": { + "guid": "df122fc3-5555-40f8-a30f-3090b8a622f8", + "typeName": "hive_table" + }, + "type": "string" + }, + "classifications": [], + "createTime": 1528238679952, + "createdBy": "anonymous", + "guid": "08bc6cee-fe25-47d2-9e39-155d4b807765", + "status": "ACTIVE", + "typeName": "hive_column", + "updateTime": 1528238690000, + "updatedBy": "anonymous", + "version": 0 + }, + "1ab9f789-b56c-4b01-a6e4-523f0fe5cdbe": { + "attributes": { + "bucketCols": null, + "compressed": false, + "inputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "location": "hdfs://localhost.localdomain:8020/apps/hive/warehouse/stocks_base.db/stocks_daily", + "numBuckets": -1, + "outputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "parameters": null, + "qualifiedName": "stocks_base.stocks_daily@cl1_storage", + "serdeInfo": { + "attributes": { + "name": null, + "parameters": { + "field.delim": ",", + "line.delim": "\n", + "serialization.format": "," + }, + "serializationLib": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe" + }, + "typeName": "hive_serde" + }, + "sortCols": null, + "storedAsSubDirectories": false, + "table": { + "guid": "df122fc3-5555-40f8-a30f-3090b8a622f8", + "typeName": "hive_table" + } + }, + "classifications": [], + "createTime": 1528238679952, + "createdBy": "anonymous", + "guid": "1ab9f789-b56c-4b01-a6e4-523f0fe5cdbe", + "status": "ACTIVE", + "typeName": "hive_storagedesc", + "updateTime": 1528238690000, + "updatedBy": "anonymous", + "version": 0 + }, + "47d732d5-a6ef-41b8-83bd-a54365750a47": { + "attributes": { + "comment": null, + "description": null, + "name": "low", + "owner": "anonymous", + "position": 3, + "qualifiedName": "stocks_base.stocks_daily.low@cl1", + "table": { + "guid": "df122fc3-5555-40f8-a30f-3090b8a622f8", + "typeName": "hive_table" + }, + "type": "string" + }, + "classifications": [], + "createTime": 1528238679952, + "createdBy": "anonymous", + "guid": "47d732d5-a6ef-41b8-83bd-a54365750a47", + "status": "ACTIVE", + "typeName": "hive_column", + "updateTime": 1528238690000, + "updatedBy": "anonymous", + "version": 0 + }, + "7193839e-10d3-4926-83d8-2c675a1068bd": { + "attributes": { + "comment": null, + "description": null, + "name": "volume", + "owner": "anonymous", + "position": 5, + "qualifiedName": "stocks_base.stocks_daily.volume@cl1", + "table": { + "guid": "df122fc3-5555-40f8-a30f-3090b8a622f8", + "typeName": "hive_table" + }, + "type": "string" + }, + "classifications": [], + "createTime": 1528238679952, + "createdBy": "anonymous", + "guid": "7193839e-10d3-4926-83d8-2c675a1068bd", + "status": "ACTIVE", + "typeName": "hive_column", + "updateTime": 1528238690000, + "updatedBy": "anonymous", + "version": 0 + }, + "92a6a9b8-dbab-4313-a6f6-0035246b0e37": { + "attributes": { + "comment": null, + "description": null, + "name": "close", + "owner": "anonymous", + "position": 4, + "qualifiedName": "stocks_base.stocks_daily.close@cl1", + "table": { + "guid": "df122fc3-5555-40f8-a30f-3090b8a622f8", + "typeName": "hive_table" + }, + "type": "string" + }, + "classifications": [], + "createTime": 1528238679952, + "createdBy": "anonymous", + "guid": "92a6a9b8-dbab-4313-a6f6-0035246b0e37", + "status": "ACTIVE", + "typeName": "hive_column", + "updateTime": 1528238690000, + "updatedBy": "anonymous", + "version": 0 + }, + "b3a13f06-4585-4fef-a5ae-970c24bd3981": { + "attributes": { + "comment": null, + "description": null, + "name": "open", + "owner": "anonymous", + "position": 1, + "qualifiedName": "stocks_base.stocks_daily.open@cl1", + "table": { + "guid": "df122fc3-5555-40f8-a30f-3090b8a622f8", + "typeName": "hive_table" + }, + "type": "string" + }, + "classifications": [], + "createTime": 1528238679952, + "createdBy": "anonymous", + "guid": "b3a13f06-4585-4fef-a5ae-970c24bd3981", + "status": "ACTIVE", + "typeName": "hive_column", + "updateTime": 1528238690000, + "updatedBy": "anonymous", + "version": 0 + }, + "c6e27ba6-9f4d-4e07-9a54-0fc631529f51": { + "attributes": { + "comment": null, + "description": null, + "name": "adj_close", + "owner": "anonymous", + "position": 6, + "qualifiedName": "stocks_base.stocks_daily.adj_close@cl1", + "table": { + "guid": "df122fc3-5555-40f8-a30f-3090b8a622f8", + "typeName": "hive_table" + }, + "type": "string" + }, + "classifications": [], + "createTime": 1528238679952, + "createdBy": "anonymous", + "guid": "c6e27ba6-9f4d-4e07-9a54-0fc631529f51", + "status": "ACTIVE", + "typeName": "hive_column", + "updateTime": 1528238690000, + "updatedBy": "anonymous", + "version": 0 + }, + "f87a5320-1529-4369-8d63-b637ebdf2c1c": { + "attributes": { + "comment": null, + "description": null, + "name": "high", + "owner": "anonymous", + "position": 2, + "qualifiedName": "stocks_base.stocks_daily.high@cl1", + "table": { + "guid": "df122fc3-5555-40f8-a30f-3090b8a622f8", + "typeName": "hive_table" + }, + "type": "string" + }, + "classifications": [], + "createTime": 1528238679952, + "createdBy": "anonymous", + "guid": "f87a5320-1529-4369-8d63-b637ebdf2c1c", + "status": "ACTIVE", + "typeName": "hive_column", + "updateTime": 1528238690000, + "updatedBy": "anonymous", + "version": 0 + } + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/resources/json/stocksDB-Entities/table-table-lineage.json ---------------------------------------------------------------------- diff --git a/repository/src/test/resources/json/stocksDB-Entities/table-table-lineage.json b/repository/src/test/resources/json/stocksDB-Entities/table-table-lineage.json new file mode 100644 index 0000000..322db8d --- /dev/null +++ b/repository/src/test/resources/json/stocksDB-Entities/table-table-lineage.json @@ -0,0 +1,44 @@ +{ + "entity": { + "attributes": { + "clusterName": "cl1", + "description": null, + "endTime": 1529605199906, + "inputs": [ + { + "guid": "df122fc3-5555-40f8-a30f-3090b8a622f8", + "typeName": "hive_table" + } + ], + "name": "create view stocks_view as select * from stocks_daily", + "operationType": "CREATEVIEW", + "outputs": [ + { + "guid": "c54400d8-79b7-45ba-9c01-b662b36ceb0e", + "typeName": "hive_table" + } + ], + "owner": null, + "qualifiedName": "stocks.stocks_view@cl1:1529605199000", + "queryGraph": null, + "queryId": "hive_20180621111959_72f4bec1-585f-4291-bc11-7a184a62a19b", + "queryPlan": "Not Supported", + "queryText": "create view stocks_view as select * from stocks_daily", + "recentQueries": [ + "create view stocks_view as select * from stocks_daily" + ], + "startTime": 1529605199002, + "userName": "anonymous" + }, + "classifications": [], + "createTime": 1529605212691, + "createdBy": "anonymous", + "guid": "6f3b305a-c459-4ae4-b651-aee0deb0685f", + "status": "ACTIVE", + "typeName": "hive_process", + "updateTime": 1529605221342, + "updatedBy": "anonymous", + "version": 0 + }, + "referredEntities": {} +} http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/resources/json/stocksDB-Entities/table-view.json ---------------------------------------------------------------------- diff --git a/repository/src/test/resources/json/stocksDB-Entities/table-view.json b/repository/src/test/resources/json/stocksDB-Entities/table-view.json new file mode 100644 index 0000000..3efa2ec --- /dev/null +++ b/repository/src/test/resources/json/stocksDB-Entities/table-view.json @@ -0,0 +1,107 @@ +{ + "entity": { + "attributes": { + "aliases": null, + "columns": [ + { + "guid": "2d3f0239-03cc-41b2-b0cb-c924ff7d096d", + "typeName": "hive_column" + } + ], + "comment": null, + "createTime": 1529605199000, + "db": { + "guid": "1637a33e-6512-447b-ade7-249c8cb5344b", + "typeName": "hive_db" + }, + "description": null, + "lastAccessTime": 1529605199000, + "name": "stocks_view", + "owner": "anonymous", + "parameters": { + "transient_lastDdlTime": "1529605199" + }, + "partitionKeys": null, + "qualifiedName": "stocks.stocks_view@cl1", + "retention": 0, + "sd": { + "guid": "56415119-7cb0-40dd-ace8-1e50efd54991", + "typeName": "hive_storagedesc" + }, + "tableType": "VIRTUAL_VIEW", + "temporary": false, + "viewExpandedText": "select `stocks_daily`.`dt`, `stocks_daily`.`open`, `stocks_daily`.`high`, `stocks_daily`.`low`, `stocks_daily`.`close`, `stocks_daily`.`volume`, `stocks_daily`.`adj_close` from `stocks`.`stocks_daily`", + "viewOriginalText": "select * from stocks_daily" + }, + "classifications": [], + "createTime": 1529605205448, + "createdBy": "anonymous", + "guid": "c54400d8-79b7-45ba-9c01-b662b36ceb0e", + "status": "ACTIVE", + "typeName": "hive_table", + "updateTime": 1529605221342, + "updatedBy": "anonymous", + "version": 0 + }, + "referredEntities": { + "2d3f0239-03cc-41b2-b0cb-c924ff7d096d": { + "attributes": { + "comment": null, + "description": null, + "name": "dt", + "owner": "anonymous", + "position": 0, + "qualifiedName": "stocks.stocks_view.dt@cl1", + "table": { + "guid": "c54400d8-79b7-45ba-9c01-b662b36ceb0e", + "typeName": "hive_table" + }, + "type": "string" + }, + "classifications": [], + "createTime": 1529605205448, + "createdBy": "anonymous", + "guid": "2d3f0239-03cc-41b2-b0cb-c924ff7d096d", + "status": "ACTIVE", + "typeName": "hive_column", + "updateTime": 1529605221342, + "updatedBy": "anonymous", + "version": 0 + }, + "56415119-7cb0-40dd-ace8-1e50efd54991": { + "attributes": { + "bucketCols": null, + "compressed": false, + "inputFormat": "org.apache.hadoop.mapred.SequenceFileInputFormat", + "location": null, + "numBuckets": -1, + "outputFormat": "org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat", + "parameters": null, + "qualifiedName": "stocks.stocks_view@cl1_storage", + "serdeInfo": { + "attributes": { + "name": null, + "parameters": null, + "serializationLib": null + }, + "typeName": "hive_serde" + }, + "sortCols": null, + "storedAsSubDirectories": false, + "table": { + "guid": "c54400d8-79b7-45ba-9c01-b662b36ceb0e", + "typeName": "hive_table" + } + }, + "classifications": [], + "createTime": 1529605205448, + "createdBy": "anonymous", + "guid": "56415119-7cb0-40dd-ace8-1e50efd54991", + "status": "ACTIVE", + "typeName": "hive_storagedesc", + "updateTime": 1529605221342, + "updatedBy": "anonymous", + "version": 0 + } + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/resources/json/stocksDB-Entities/typesdef-new-classification.json ---------------------------------------------------------------------- diff --git a/repository/src/test/resources/json/stocksDB-Entities/typesdef-new-classification.json b/repository/src/test/resources/json/stocksDB-Entities/typesdef-new-classification.json new file mode 100644 index 0000000..3aac4ea --- /dev/null +++ b/repository/src/test/resources/json/stocksDB-Entities/typesdef-new-classification.json @@ -0,0 +1,22 @@ +{ + "classificationDefs": [ + { + "attributeDefs": [], + "category": "CLASSIFICATION", + "createTime": 1528240995728, + "createdBy": "admin", + "description": "T1", + "guid": "29fc3eed-e97a-429c-ad9e-45d6ccb99fce", + "name": "T1", + "subTypes": [], + "superTypes": [], + "typeVersion": "1.0", + "updateTime": 1528240995728, + "updatedBy": "admin", + "version": 1 + } + ], + "entityDefs": [], + "enumDefs": [], + "structDefs": [] +}
