ATLAS-385 Support for Lineage for entities with SuperType as DataSet (anilsg via sumasai)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/ae1d636f Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/ae1d636f Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/ae1d636f Branch: refs/heads/branch-0.6-incubating Commit: ae1d636fbb48346f4cfc926be469a6bfc884f71e Parents: 8cbb2c1 Author: Suma Shivaprasad <[email protected]> Authored: Thu Dec 17 21:56:03 2015 +0530 Committer: Suma Shivaprasad <[email protected]> Committed: Thu Dec 17 21:56:03 2015 +0530 ---------------------------------------------------------------------- dashboard/public/css/lineage.css | 1 + .../public/modules/details/detailsController.js | 10 +- .../public/modules/details/views/details.html | 2 +- .../modules/lineage/lineage_ioController.js | 131 ++++++------- release-log.txt | 1 + ...kedMetadataRepositoryDeleteEntitiesTest.java | 189 +++++++++++++++++++ 6 files changed, 254 insertions(+), 80 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ae1d636f/dashboard/public/css/lineage.css ---------------------------------------------------------------------- diff --git a/dashboard/public/css/lineage.css b/dashboard/public/css/lineage.css index d700e34..180bebf 100644 --- a/dashboard/public/css/lineage.css +++ b/dashboard/public/css/lineage.css @@ -86,4 +86,5 @@ div.lineage { .alignLineage{ text-align: center; margin-top: 100px; + font-size: 24px; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ae1d636f/dashboard/public/modules/details/detailsController.js ---------------------------------------------------------------------- diff --git a/dashboard/public/modules/details/detailsController.js b/dashboard/public/modules/details/detailsController.js index c03824c..cc6bb2d 100644 --- a/dashboard/public/modules/details/detailsController.js +++ b/dashboard/public/modules/details/detailsController.js @@ -22,14 +22,14 @@ angular.module('dgc.details').controller('DetailsController', ['$window', '$scop $scope.tableName = false; $scope.isTable = false; - + $scope.isLineage = false; + DetailsResource.get({ id: $stateParams.id }, function(data) { $scope.details = data; $scope.tableName = data.values.name; - $scope.isTable = (typeof data.typeName !== 'undefined' && data.typeName.toLowerCase().indexOf('table') !== -1) ? true : false; $scope.onActivate('io'); $scope.isTags = (typeof data.traits !== 'undefined' && typeof data.traits === 'object') ? true : false; @@ -56,6 +56,9 @@ angular.module('dgc.details').controller('DetailsController', ['$window', '$scop } }); + $scope.$on('show_lineage', function() { + $scope.isLineage = true; + }); $scope.isNumber = angular.isNumber; $scope.isObject = angular.isObject; @@ -64,7 +67,8 @@ angular.module('dgc.details').controller('DetailsController', ['$window', '$scop $scope.onActivate = function tabActivate(tabname) { $scope.$broadcast('render-lineage', { type: tabname, - tableName: $scope.tableName + tableName: $scope.tableName, + guid : $stateParams.id }); }; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ae1d636f/dashboard/public/modules/details/views/details.html ---------------------------------------------------------------------- diff --git a/dashboard/public/modules/details/views/details.html b/dashboard/public/modules/details/views/details.html index 0878308..c84aeb2 100644 --- a/dashboard/public/modules/details/views/details.html +++ b/dashboard/public/modules/details/views/details.html @@ -28,7 +28,7 @@ <h4 ng-if="details.values && details.values.name && details.values.name != ''"> <b>Name:</b> <span class="black">{{details.values.name}}</span></h2> <h4 ng-if="details.values && details.values.description && details.values.description != ''"><b>Description:</b> <span class="black">{{details.values.description}}</span></h4> - <h4 data-ng-show="isTable" data-disable="!tableName" data-select="onActivate('io')"> + <h4 data-disable="!tableName" data-select="onActivate('io')" id="lineageGraph" class="hide"> <span class="lineage">Lineage</span> <ng-include data-table-type="io" src="'/modules/lineage/views/lineage_io.html'"/> </h4> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ae1d636f/dashboard/public/modules/lineage/lineage_ioController.js ---------------------------------------------------------------------- diff --git a/dashboard/public/modules/lineage/lineage_ioController.js b/dashboard/public/modules/lineage/lineage_ioController.js index e7b2734..bb8fcec 100644 --- a/dashboard/public/modules/lineage/lineage_ioController.js +++ b/dashboard/public/modules/lineage/lineage_ioController.js @@ -39,51 +39,64 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element', '$ LineageResource.get({ tableName: tableData.tableName, type: 'outputs' - }, function lineageSuccess(response1) { - - LineageResource.get({ - tableName: tableData.tableName, - type: 'inputs' - }, function lineageSuccess(response) { - response.results.values.edges = inVertObj(response.results.values.edges); - - angular.forEach(response.results.values.edges, function(value, key) { - angular.forEach(response1.results.values.edges, function(value1, key1) { - if (key === key1) { - var array1 = value; - angular.forEach(value1, function(value2) { - array1.push(value2); + }).$promise.then( + function lineageSuccess(response1) { + // $scope.$emit('show_lineage'); + $('#lineageGraph').removeClass('hide'); + LineageResource.get({ + tableName: tableData.tableName, + type: 'inputs' + }).$promise.then( + //success + function lineageSuccess(response) { + if (response && response.results) { + response.results.values.edges = inVertObj(response.results.values.edges); + + angular.forEach(response.results.values.edges, function(value, key) { + angular.forEach(response1.results.values.edges, function(value1, key1) { + if (key === key1) { + var array1 = value; + angular.forEach(value1, function(value2) { + array1.push(value2); + }); + response.results.values.edges[key] = array1; + response1.results.values.edges[key] = array1; + } + }); }); - response.results.values.edges[key] = array1; - response1.results.values.edges[key] = array1; - } - }); - }); - angular.extend(response.results.values.edges, response1.results.values.edges); - angular.extend(response.results.values.vertices, response1.results.values.vertices); + angular.extend(response.results.values.edges, response1.results.values.edges); + angular.extend(response.results.values.vertices, response1.results.values.vertices); - if (!_.isEmpty(response.results.values.vertices)) { - loadProcess(response.results.values.edges, response.results.values.vertices) - .then(function(res) { - guidsList = res; + if (!_.isEmpty(response.results.values.vertices)) { + loadProcess(response.results.values.edges, response.results.values.vertices) + .then(function(res) { + guidsList = res; - $scope.lineageData = transformData(response.results); + $scope.lineageData = transformData(response.results); - if (callRender) { - render(); + if (callRender) { + render(); + } + }); + } else { + $scope.requested = false; } - }); - } else { - $scope.requested = false; - } - }); - - }); - + } else { + $scope.requested = false; + } + }, + function() { + $scope.requested = false; + } + ); + }, + function() { + $scope.requested = false; + } + ); } - function loadProcess(edges, vertices) { var urlCalls = []; @@ -141,6 +154,7 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element', '$ render(); } } + $scope.guid = lineageData.guid; }); function transformData(metaData) { @@ -157,11 +171,11 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element', '$ var loadProcess = getLoadProcessTypes(guid); if (typeof loadProcess !== "undefined") { name = loadProcess.name; - type = loadProcess.typeName; + type = 'edges'; tip = loadProcess.tip; } else { name = 'Load Process'; - type = 'Load Process'; + type = 'edges'; } } var vertex = { @@ -433,29 +447,6 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element', '$ zoomListener.scale(scale); zoomListener.translate([x, y]); } - - // Toggle children function - - // function toggleChildren(d) { - // if (d.children) { - // d._children = d.children; - // d.children = null; - // } else if (d._children) { - // d.children = d._children; - // d._children = null; - // } - // return d; - // } - - // Toggle children on click. - - // function click(d) { - // if (d3.event.defaultPrevented) return; // click suppressed - // d = toggleChildren(d); - // update(d); - // //centerNode(d); - // } - //arrow baseSvg.append("svg:defs") .append("svg:marker") @@ -568,10 +559,10 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element', '$ nodeEnter.append("image") .attr("class", "nodeImage") .attr("xlink:href", function(d) { - return d.type === 'Table' ? '../img/tableicon.png' : '../img/process.png'; + return (d.type && d.type !== '' && d.type.toLowerCase().indexOf('edges') !== -1) ? '../img/process.png' : '../img/tableicon.png'; }) .on('mouseover', function(d) { - if (d.type === 'LoadProcess' || 'Table') { + if (d.type === 'edges' || 'Table') { tooltip.show(d); } }) @@ -623,18 +614,6 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element', '$ return nameDis; }); - // Change the circle fill depending on whether it has children and is collapsed - // Change the circle fill depending on whether it has children and is collapsed - node.select("image.nodeImage") - .attr("r", 4.5) - .attr("xlink:href", function(d) { - if (d._children) { - return d.type === 'Table' ? '../img/tableicon1.png' : '../img/process1.png'; - } - return d.type === 'Table' ? '../img/tableicon.png' : '../img/process.png'; - }); - - // Transition nodes to their new position. var nodeUpdate = node.transition() .duration(duration) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ae1d636f/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index ce3337d..7d45a7b 100644 --- a/release-log.txt +++ b/release-log.txt @@ -14,6 +14,7 @@ ATLAS-54 Rename configs in hive hook (shwethags) ATLAS-3 Mixed Index creation fails with Date types (sumasai via shwethags) ALL CHANGES: +ATLAS-385 Support for Lineage for entities with SuperType as DataSet (anilsg via sumasai) ATLAS-342 Atlas is sending an ENTITY_CREATE event to the ATLAS_ENTITIES topic even if the entity exists already (shwethags) ATLAS-386 Handle hive rename Table (shwethags) ATLAS-374 Doc: Create a wiki for documenting fault tolerance and HA options for Atlas data (yhemanth via sumasai) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ae1d636f/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java new file mode 100644 index 0000000..e63f6d3 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java @@ -0,0 +1,189 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.graph; + +import java.util.ArrayList; +import java.util.List; + +import javax.inject.Inject; + +import org.apache.atlas.RepositoryMetadataModule; +import org.apache.atlas.TestUtils; +import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; +import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.RepositoryException; +import org.apache.atlas.typesystem.ITypedReferenceableInstance; +import org.apache.atlas.typesystem.Referenceable; +import org.apache.atlas.typesystem.exception.EntityNotFoundException; +import org.apache.atlas.typesystem.types.ClassType; +import org.apache.atlas.typesystem.types.Multiplicity; +import org.apache.atlas.typesystem.types.TypeSystem; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Guice; +import org.testng.annotations.Test; + +import com.thinkaurelius.titan.core.TitanGraph; +import com.thinkaurelius.titan.core.util.TitanCleanup; +import com.tinkerpop.blueprints.Vertex; + +/** + * Test for GraphBackedMetadataRepository.deleteEntities + * + * Guice loads the dependencies and injects the necessary objects + * + */ +@Guice(modules = RepositoryMetadataModule.class) +public class GraphBackedMetadataRepositoryDeleteEntitiesTest { + + @Inject + private GraphProvider<TitanGraph> graphProvider; + + @Inject + private GraphBackedMetadataRepository repositoryService; + + @Inject + private GraphBackedDiscoveryService discoveryService; + + private TypeSystem typeSystem; + + @BeforeClass + public void setUp() throws Exception { + typeSystem = TypeSystem.getInstance(); + typeSystem.reset(); + + new GraphBackedSearchIndexer(graphProvider); + + TestUtils.defineDeptEmployeeTypes(typeSystem); + TestUtils.createHiveTypes(typeSystem); + } + + + @AfterClass + public void tearDown() throws Exception { + TypeSystem.getInstance().reset(); + try { + graphProvider.get().shutdown(); + } catch (Exception e) { + e.printStackTrace(); + } + try { + TitanCleanup.clear(graphProvider.get()); + } catch (Exception e) { + e.printStackTrace(); + } + } + + + + @Test + public void testDeleteEntities() throws Exception { + String hrDeptGuid = createHrDeptGraph(); + + ITypedReferenceableInstance hrDept = repositoryService.getEntityDefinition(hrDeptGuid); + Object refValue = hrDept.get("employees"); + Assert.assertTrue(refValue instanceof List); + List<Object> employees = (List<Object>)refValue; + Assert.assertEquals(employees.size(), 4); + List<String> employeeGuids = new ArrayList<String>(4); + for (Object listValue : employees) { + Assert.assertTrue(listValue instanceof ITypedReferenceableInstance); + ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue; + employeeGuids.add(employee.getId()._getId()); + } + + // There should be 4 vertices for Address structs (one for each Person.address attribute value). + int vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "Address"); + Assert.assertEquals(vertexCount, 4); + vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "SecurityClearance"); + Assert.assertEquals(vertexCount, 1); + + List<String> deletedEntities = repositoryService.deleteEntities(hrDeptGuid); + Assert.assertTrue(deletedEntities.contains(hrDeptGuid)); + + // Verify Department entity and its contained Person entities were deleted. + verifyEntityDoesNotExist(hrDeptGuid); + for (String employeeGuid : employeeGuids) { + verifyEntityDoesNotExist(employeeGuid); + } + // Verify all Person.address struct vertices were removed. + vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "Address"); + Assert.assertEquals(vertexCount, 0); + + // Verify all SecurityClearance trait vertices were removed. + vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "SecurityClearance"); + Assert.assertEquals(vertexCount, 0); + } + + @Test(dependsOnMethods = "testDeleteEntities") + public void testDeleteContainedEntity() throws Exception { + String hrDeptGuid = createHrDeptGraph(); + ITypedReferenceableInstance hrDept = repositoryService.getEntityDefinition(hrDeptGuid); + Object refValue = hrDept.get("employees"); + Assert.assertTrue(refValue instanceof List); + List<Object> employees = (List<Object>)refValue; + Assert.assertEquals(employees.size(), 4); + Object listValue = employees.get(2); + Assert.assertTrue(listValue instanceof ITypedReferenceableInstance); + ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue; + String employeeGuid = employee.getId()._getId(); + + List<String> deletedEntities = repositoryService.deleteEntities(employeeGuid); + Assert.assertTrue(deletedEntities.contains(employeeGuid)); + verifyEntityDoesNotExist(employeeGuid); + + } + + private String createHrDeptGraph() throws Exception { + Referenceable deptEg1 = TestUtils.createDeptEg1(typeSystem); + ClassType deptType = typeSystem.getDataType(ClassType.class, "Department"); + ITypedReferenceableInstance hrDept2 = deptType.convert(deptEg1, Multiplicity.REQUIRED); + + List<String> guids = repositoryService.createEntities(hrDept2); + Assert.assertNotNull(guids); + Assert.assertEquals(guids.size(), 5); + List<String> entityList = repositoryService.getEntityList("Department"); + Assert.assertNotNull(entityList); + Assert.assertEquals(entityList.size(), 1); + String hrDeptGuid = entityList.get(0); + return hrDeptGuid; + } + + private int countVertices(String propertyName, Object value) { + Iterable<Vertex> vertices = graphProvider.get().getVertices(propertyName, value); + int vertexCount = 0; + for (Vertex vertex : vertices) { + vertexCount++; + } + return vertexCount; + } + + private void verifyEntityDoesNotExist(String hrDeptGuid) throws RepositoryException { + + try { + repositoryService.getEntityDefinition(hrDeptGuid); + Assert.fail("EntityNotFoundException was expected but none thrown"); + } + catch(EntityNotFoundException e) { + // good + } + } + +}
