atlas git commit: ATLAS-2927: Update lineage query for Process entities #2
Repository: atlas
Updated Branches:
refs/heads/branch-1.0 7aebca8ad -> d426815ca
ATLAS-2927: Update lineage query for Process entities #2
(cherry picked from commit da78c48314c67f2998eab2190324ca3786b3bafc)
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/d426815c
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/d426815c
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/d426815c
Branch: refs/heads/branch-1.0
Commit: d426815ca7a494585718663edb9ebf92c3e14f4a
Parents: 7aebca8
Author: Sarath Subramanian
Authored: Fri Oct 26 16:02:55 2018 -0700
Committer: Sarath Subramanian
Committed: Fri Oct 26 16:03:41 2018 -0700
--
.../java/org/apache/atlas/discovery/EntityLineageService.java| 3 ++-
.../java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java | 4 ++--
2 files changed, 4 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/d426815c/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
index 89c969b..9a02046 100644
---
a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
+++
b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
@@ -309,7 +309,8 @@ public class EntityLineageService implements
AtlasLineageService {
bindings.put("guid", entityGuid);
bindings.put("incomingEdgeLabel", incomingFrom);
bindings.put("outgoingEdgeLabel", outgoingTo);
-bindings.put("depth", depth);
+bindings.put("dataSetDepth", depth);
+bindings.put("processDepth", depth - 1);
if (depth < 1) {
ret = isDataSet ?
gremlinQueryProvider.getQuery(FULL_LINEAGE_DATASET) :
http://git-wip-us.apache.org/repos/asf/atlas/blob/d426815c/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
index 6d3b1a8..20c570f 100644
---
a/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
+++
b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
@@ -48,11 +48,11 @@ public class AtlasGremlin3QueryProvider extends
AtlasGremlin2QueryProvider {
case FULL_LINEAGE_DATASET:
return "g.V().has('__guid',
guid).repeat(__.inE(incomingEdgeLabel).as('e1').outV().outE(outgoingEdgeLabel).as('e2').inV()).emit().select('e1',
'e2').toList()";
case PARTIAL_LINEAGE_DATASET:
-return "g.V().has('__guid',
guid).repeat(__.inE(incomingEdgeLabel).as('e1').outV().outE(outgoingEdgeLabel).as('e2').inV()).times(depth).emit().select('e1',
'e2').toList()";
+return "g.V().has('__guid',
guid).repeat(__.inE(incomingEdgeLabel).as('e1').outV().outE(outgoingEdgeLabel).as('e2').inV()).times(dataSetDepth).emit().select('e1',
'e2').toList()";
case FULL_LINEAGE_PROCESS:
return "g.V().has('__guid',
guid).outE(outgoingEdgeLabel).store('e').inV().repeat(__.inE(incomingEdgeLabel).store('e').outV().outE(outgoingEdgeLabel).store('e').inV()).cap('e').unfold().toList()";
case PARTIAL_LINEAGE_PROCESS:
-return "g.V().has('__guid',
guid).outE(outgoingEdgeLabel).store('e').inV().repeat(__.inE(incomingEdgeLabel).store('e').outV().outE(outgoingEdgeLabel).store('e').inV()).times(depth).cap('e').unfold().toList()";
+return "g.V().has('__guid',
guid).outE(outgoingEdgeLabel).store('e').inV().until(loops().is(eq(processDepth))).repeat(__.inE(incomingEdgeLabel).store('e').outV().outE(outgoingEdgeLabel).store('e').inV()).cap('e').unfold().toList()";
case TO_RANGE_LIST:
return ".range(startIdx, endIdx).toList()";
case RELATIONSHIP_SEARCH:
atlas git commit: ATLAS-2927: Update lineage query for Process entities #2
Repository: atlas
Updated Branches:
refs/heads/master 572c5d648 -> da78c4831
ATLAS-2927: Update lineage query for Process entities #2
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/da78c483
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/da78c483
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/da78c483
Branch: refs/heads/master
Commit: da78c48314c67f2998eab2190324ca3786b3bafc
Parents: 572c5d6
Author: Sarath Subramanian
Authored: Fri Oct 26 16:02:55 2018 -0700
Committer: Sarath Subramanian
Committed: Fri Oct 26 16:02:55 2018 -0700
--
.../java/org/apache/atlas/discovery/EntityLineageService.java| 3 ++-
.../java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java | 4 ++--
2 files changed, 4 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/da78c483/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
index 89c969b..9a02046 100644
---
a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
+++
b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
@@ -309,7 +309,8 @@ public class EntityLineageService implements
AtlasLineageService {
bindings.put("guid", entityGuid);
bindings.put("incomingEdgeLabel", incomingFrom);
bindings.put("outgoingEdgeLabel", outgoingTo);
-bindings.put("depth", depth);
+bindings.put("dataSetDepth", depth);
+bindings.put("processDepth", depth - 1);
if (depth < 1) {
ret = isDataSet ?
gremlinQueryProvider.getQuery(FULL_LINEAGE_DATASET) :
http://git-wip-us.apache.org/repos/asf/atlas/blob/da78c483/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
index 6d3b1a8..20c570f 100644
---
a/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
+++
b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
@@ -48,11 +48,11 @@ public class AtlasGremlin3QueryProvider extends
AtlasGremlin2QueryProvider {
case FULL_LINEAGE_DATASET:
return "g.V().has('__guid',
guid).repeat(__.inE(incomingEdgeLabel).as('e1').outV().outE(outgoingEdgeLabel).as('e2').inV()).emit().select('e1',
'e2').toList()";
case PARTIAL_LINEAGE_DATASET:
-return "g.V().has('__guid',
guid).repeat(__.inE(incomingEdgeLabel).as('e1').outV().outE(outgoingEdgeLabel).as('e2').inV()).times(depth).emit().select('e1',
'e2').toList()";
+return "g.V().has('__guid',
guid).repeat(__.inE(incomingEdgeLabel).as('e1').outV().outE(outgoingEdgeLabel).as('e2').inV()).times(dataSetDepth).emit().select('e1',
'e2').toList()";
case FULL_LINEAGE_PROCESS:
return "g.V().has('__guid',
guid).outE(outgoingEdgeLabel).store('e').inV().repeat(__.inE(incomingEdgeLabel).store('e').outV().outE(outgoingEdgeLabel).store('e').inV()).cap('e').unfold().toList()";
case PARTIAL_LINEAGE_PROCESS:
-return "g.V().has('__guid',
guid).outE(outgoingEdgeLabel).store('e').inV().repeat(__.inE(incomingEdgeLabel).store('e').outV().outE(outgoingEdgeLabel).store('e').inV()).times(depth).cap('e').unfold().toList()";
+return "g.V().has('__guid',
guid).outE(outgoingEdgeLabel).store('e').inV().until(loops().is(eq(processDepth))).repeat(__.inE(incomingEdgeLabel).store('e').outV().outE(outgoingEdgeLabel).store('e').inV()).cap('e').unfold().toList()";
case TO_RANGE_LIST:
return ".range(startIdx, endIdx).toList()";
case RELATIONSHIP_SEARCH:
atlas git commit: ATLAS-2927: Update lineage query for Process entities
Repository: atlas
Updated Branches:
refs/heads/branch-0.8 707bcb1cb -> 2a547434b
ATLAS-2927: Update lineage query for Process entities
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/2a547434
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/2a547434
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/2a547434
Branch: refs/heads/branch-0.8
Commit: 2a547434b5dd5a8a78ed915f17b05b674aa9df95
Parents: 707bcb1
Author: Sarath Subramanian
Authored: Wed Oct 24 11:10:58 2018 -0700
Committer: Sarath Subramanian
Committed: Wed Oct 24 11:10:58 2018 -0700
--
.../atlas/discovery/EntityLineageService.java | 124 +++
.../atlas/util/AtlasGremlin2QueryProvider.java | 10 +-
2 files changed, 78 insertions(+), 56 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/2a547434/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
index c928d35..472d692 100644
---
a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
+++
b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
@@ -26,8 +26,10 @@ import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.lineage.AtlasLineageInfo;
import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection;
import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageRelation;
+import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
import org.apache.atlas.repository.store.graph.v1.EntityGraphRetriever;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasTypeRegistry;
@@ -36,6 +38,8 @@ import org.apache.commons.collections.CollectionUtils;
import org.springframework.stereotype.Service;
import javax.inject.Inject;
+import javax.script.ScriptEngine;
+import javax.script.ScriptException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -45,6 +49,7 @@ import java.util.Set;
import static org.apache.atlas.AtlasClient.DATA_SET_SUPER_TYPE;
import static org.apache.atlas.AtlasClient.PROCESS_SUPER_TYPE;
+import static org.apache.atlas.AtlasErrorCode.INSTANCE_LINEAGE_QUERY_FAILED;
import static
org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection.BOTH;
import static
org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection.INPUT;
import static
org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection.OUTPUT;
@@ -114,39 +119,14 @@ public class EntityLineageService implements
AtlasLineageService {
private AtlasLineageInfo getLineageInfo(String guid, LineageDirection
direction, int depth, boolean isDataSet) throws AtlasBaseException {
Map entities = new HashMap<>();
Set relations= new HashSet<>();
-String lineageQuery = getLineageQuery(guid,
direction, depth, isDataSet);
-
-List paths = (List) graph.executeGremlinScript(lineageQuery, true);
-
-if (CollectionUtils.isNotEmpty(paths)) {
-for (Object path : paths) {
-if (path instanceof List) {
-List vertices = (List) path;
-
-if (CollectionUtils.isNotEmpty(vertices)) {
-AtlasEntityHeader prev = null;
-
-for (Object vertex : vertices) {
-if (!(vertex instanceof AtlasVertex)) {
-continue;
-}
-
-AtlasEntityHeader entity =
entityRetriever.toAtlasEntityHeader((AtlasVertex)vertex);
-
-if (!entities.containsKey(entity.getGuid())) {
-entities.put(entity.getGuid(), entity);
-}
-
-if (prev != null) {
-if (direction.equals(INPUT)) {
-relations.add(new
LineageRelation(entity.getGuid(), prev.getGuid()));
-} else if (direction.equals(OUTPUT)) {
-relations.add(new
LineageRelation(prev.getGuid(), entity.getGuid()));
-}
-}
-prev = entity;
-}
-}
+final Map bindings = new
atlas git commit: ATLAS-2927: Update lineage query for Process entities
Repository: atlas
Updated Branches:
refs/heads/branch-1.0 23e61364f -> 9653cc5d5
ATLAS-2927: Update lineage query for Process entities
(cherry picked from commit 46b9b7c85835b1c4285eddce6c9773024a1b2114)
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/9653cc5d
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/9653cc5d
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/9653cc5d
Branch: refs/heads/branch-1.0
Commit: 9653cc5d544ce07ffe236462ee09a32d58f75011
Parents: 23e6136
Author: Sarath Subramanian
Authored: Sun Oct 21 22:22:38 2018 -0700
Committer: Sarath Subramanian
Committed: Sun Oct 21 22:24:32 2018 -0700
--
.../atlas/discovery/EntityLineageService.java | 68 +---
.../atlas/util/AtlasGremlin3QueryProvider.java | 8 +--
2 files changed, 49 insertions(+), 27 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/9653cc5d/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
index 6f2f97b..89c969b 100644
---
a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
+++
b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
@@ -41,7 +41,6 @@ import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.atlas.util.AtlasGremlinQueryProvider;
-import org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery;
import org.apache.atlas.v1.model.lineage.SchemaResponse.SchemaDetails;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
@@ -51,6 +50,8 @@ import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import javax.inject.Inject;
+import javax.script.ScriptEngine;
+import javax.script.ScriptException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -61,6 +62,7 @@ import java.util.stream.Collectors;
import static org.apache.atlas.AtlasClient.DATA_SET_SUPER_TYPE;
import static org.apache.atlas.AtlasClient.PROCESS_SUPER_TYPE;
+import static org.apache.atlas.AtlasErrorCode.INSTANCE_LINEAGE_QUERY_FAILED;
import static
org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection.BOTH;
import static
org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection.INPUT;
import static
org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection.OUTPUT;
@@ -203,16 +205,16 @@ public class EntityLineageService implements
AtlasLineageService {
}
private AtlasLineageInfo getLineageInfo(String guid, LineageDirection
direction, int depth, boolean isDataSet) throws AtlasBaseException {
+final Map bindings = new HashMap<>();
+String lineageQuery = getLineageQuery(guid,
direction, depth, isDataSet, bindings);
+List results =
executeGremlinScript(bindings, lineageQuery);
Map entities = new HashMap<>();
Set relations= new HashSet<>();
-String lineageQuery = getLineageQuery(guid,
direction, depth, isDataSet);
-List edgeMapList = (List) graph.executeGremlinScript(lineageQuery,
false);
-
-if (CollectionUtils.isNotEmpty(edgeMapList)) {
-for (Object edgeMap : edgeMapList) {
-if (edgeMap instanceof Map) {
-for (final Object o : ((Map) edgeMap).entrySet()) {
+if (CollectionUtils.isNotEmpty(results)) {
+for (Object result : results) {
+if (result instanceof Map) {
+for (final Object o : ((Map) result).entrySet()) {
final Map.Entry entry = (Map.Entry) o;
Object value = entry.getValue();
@@ -230,6 +232,8 @@ public class EntityLineageService implements
AtlasLineageService {
LOG.warn("Invalid value of type {} found,
ignoring", (value != null ? value.getClass().getSimpleName() : "null"));
}
}
+} else if (result instanceof AtlasEdge) {
+processEdge((AtlasEdge) result, entities, relations);
}
}
}
@@ -237,6 +241,21 @@ public class EntityLineageService implements
AtlasLineageService {
return new AtlasLineageInfo(guid, entities, relations, direction,
depth);
}
+private List executeGremlinScript(Map bindings, String
atlas git commit: ATLAS-2927: Update lineage query for Process entities
Repository: atlas
Updated Branches:
refs/heads/master 3b8a34c51 -> 46b9b7c85
ATLAS-2927: Update lineage query for Process entities
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/46b9b7c8
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/46b9b7c8
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/46b9b7c8
Branch: refs/heads/master
Commit: 46b9b7c85835b1c4285eddce6c9773024a1b2114
Parents: 3b8a34c
Author: Sarath Subramanian
Authored: Sun Oct 21 22:22:38 2018 -0700
Committer: Sarath Subramanian
Committed: Sun Oct 21 22:22:38 2018 -0700
--
.../atlas/discovery/EntityLineageService.java | 68 +---
.../atlas/util/AtlasGremlin3QueryProvider.java | 8 +--
2 files changed, 49 insertions(+), 27 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/46b9b7c8/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
index 6f2f97b..89c969b 100644
---
a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
+++
b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
@@ -41,7 +41,6 @@ import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.atlas.util.AtlasGremlinQueryProvider;
-import org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery;
import org.apache.atlas.v1.model.lineage.SchemaResponse.SchemaDetails;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
@@ -51,6 +50,8 @@ import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import javax.inject.Inject;
+import javax.script.ScriptEngine;
+import javax.script.ScriptException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -61,6 +62,7 @@ import java.util.stream.Collectors;
import static org.apache.atlas.AtlasClient.DATA_SET_SUPER_TYPE;
import static org.apache.atlas.AtlasClient.PROCESS_SUPER_TYPE;
+import static org.apache.atlas.AtlasErrorCode.INSTANCE_LINEAGE_QUERY_FAILED;
import static
org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection.BOTH;
import static
org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection.INPUT;
import static
org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection.OUTPUT;
@@ -203,16 +205,16 @@ public class EntityLineageService implements
AtlasLineageService {
}
private AtlasLineageInfo getLineageInfo(String guid, LineageDirection
direction, int depth, boolean isDataSet) throws AtlasBaseException {
+final Map bindings = new HashMap<>();
+String lineageQuery = getLineageQuery(guid,
direction, depth, isDataSet, bindings);
+List results =
executeGremlinScript(bindings, lineageQuery);
Map entities = new HashMap<>();
Set relations= new HashSet<>();
-String lineageQuery = getLineageQuery(guid,
direction, depth, isDataSet);
-List edgeMapList = (List) graph.executeGremlinScript(lineageQuery,
false);
-
-if (CollectionUtils.isNotEmpty(edgeMapList)) {
-for (Object edgeMap : edgeMapList) {
-if (edgeMap instanceof Map) {
-for (final Object o : ((Map) edgeMap).entrySet()) {
+if (CollectionUtils.isNotEmpty(results)) {
+for (Object result : results) {
+if (result instanceof Map) {
+for (final Object o : ((Map) result).entrySet()) {
final Map.Entry entry = (Map.Entry) o;
Object value = entry.getValue();
@@ -230,6 +232,8 @@ public class EntityLineageService implements
AtlasLineageService {
LOG.warn("Invalid value of type {} found,
ignoring", (value != null ? value.getClass().getSimpleName() : "null"));
}
}
+} else if (result instanceof AtlasEdge) {
+processEdge((AtlasEdge) result, entities, relations);
}
}
}
@@ -237,6 +241,21 @@ public class EntityLineageService implements
AtlasLineageService {
return new AtlasLineageInfo(guid, entities, relations, direction,
depth);
}
+private List executeGremlinScript(Map bindings, String
lineageQuery) throws AtlasBaseException {
+List ret;
+
