atlas git commit: ATLAS-2927: Update lineage query for Process entities #2

2018-10-26 Thread sarath 
Repository: atlas
Updated Branches:
  refs/heads/branch-1.0 7aebca8ad -> d426815ca


ATLAS-2927: Update lineage query for Process entities #2

(cherry picked from commit da78c48314c67f2998eab2190324ca3786b3bafc)


Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/d426815c
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/d426815c
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/d426815c

Branch: refs/heads/branch-1.0
Commit: d426815ca7a494585718663edb9ebf92c3e14f4a
Parents: 7aebca8
Author: Sarath Subramanian 
Authored: Fri Oct 26 16:02:55 2018 -0700
Committer: Sarath Subramanian 
Committed: Fri Oct 26 16:03:41 2018 -0700

--
 .../java/org/apache/atlas/discovery/EntityLineageService.java| 3 ++-
 .../java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java   | 4 ++--
 2 files changed, 4 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/atlas/blob/d426815c/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
--
diff --git 
a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java 
b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
index 89c969b..9a02046 100644
--- 
a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
+++ 
b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
@@ -309,7 +309,8 @@ public class EntityLineageService implements 
AtlasLineageService {
 bindings.put("guid", entityGuid);
 bindings.put("incomingEdgeLabel", incomingFrom);
 bindings.put("outgoingEdgeLabel", outgoingTo);
-bindings.put("depth", depth);
+bindings.put("dataSetDepth", depth);
+bindings.put("processDepth", depth - 1);
 
 if (depth < 1) {
 ret = isDataSet ? 
gremlinQueryProvider.getQuery(FULL_LINEAGE_DATASET) :

http://git-wip-us.apache.org/repos/asf/atlas/blob/d426815c/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
--
diff --git 
a/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
 
b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
index 6d3b1a8..20c570f 100644
--- 
a/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
+++ 
b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
@@ -48,11 +48,11 @@ public class AtlasGremlin3QueryProvider extends 
AtlasGremlin2QueryProvider {
 case FULL_LINEAGE_DATASET:
 return "g.V().has('__guid', 
guid).repeat(__.inE(incomingEdgeLabel).as('e1').outV().outE(outgoingEdgeLabel).as('e2').inV()).emit().select('e1',
 'e2').toList()";
 case PARTIAL_LINEAGE_DATASET:
-return "g.V().has('__guid', 
guid).repeat(__.inE(incomingEdgeLabel).as('e1').outV().outE(outgoingEdgeLabel).as('e2').inV()).times(depth).emit().select('e1',
 'e2').toList()";
+return "g.V().has('__guid', 
guid).repeat(__.inE(incomingEdgeLabel).as('e1').outV().outE(outgoingEdgeLabel).as('e2').inV()).times(dataSetDepth).emit().select('e1',
 'e2').toList()";
 case FULL_LINEAGE_PROCESS:
 return "g.V().has('__guid', 
guid).outE(outgoingEdgeLabel).store('e').inV().repeat(__.inE(incomingEdgeLabel).store('e').outV().outE(outgoingEdgeLabel).store('e').inV()).cap('e').unfold().toList()";
 case PARTIAL_LINEAGE_PROCESS:
-return "g.V().has('__guid', 
guid).outE(outgoingEdgeLabel).store('e').inV().repeat(__.inE(incomingEdgeLabel).store('e').outV().outE(outgoingEdgeLabel).store('e').inV()).times(depth).cap('e').unfold().toList()";
+return "g.V().has('__guid', 
guid).outE(outgoingEdgeLabel).store('e').inV().until(loops().is(eq(processDepth))).repeat(__.inE(incomingEdgeLabel).store('e').outV().outE(outgoingEdgeLabel).store('e').inV()).cap('e').unfold().toList()";
 case TO_RANGE_LIST:
 return ".range(startIdx, endIdx).toList()";
 case RELATIONSHIP_SEARCH:

atlas git commit: ATLAS-2927: Update lineage query for Process entities #2

2018-10-26 Thread sarath 
Repository: atlas
Updated Branches:
  refs/heads/master 572c5d648 -> da78c4831


ATLAS-2927: Update lineage query for Process entities #2


Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/da78c483
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/da78c483
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/da78c483

Branch: refs/heads/master
Commit: da78c48314c67f2998eab2190324ca3786b3bafc
Parents: 572c5d6
Author: Sarath Subramanian 
Authored: Fri Oct 26 16:02:55 2018 -0700
Committer: Sarath Subramanian 
Committed: Fri Oct 26 16:02:55 2018 -0700

--
 .../java/org/apache/atlas/discovery/EntityLineageService.java| 3 ++-
 .../java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java   | 4 ++--
 2 files changed, 4 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/atlas/blob/da78c483/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
--
diff --git 
a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java 
b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
index 89c969b..9a02046 100644
--- 
a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
+++ 
b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
@@ -309,7 +309,8 @@ public class EntityLineageService implements 
AtlasLineageService {
 bindings.put("guid", entityGuid);
 bindings.put("incomingEdgeLabel", incomingFrom);
 bindings.put("outgoingEdgeLabel", outgoingTo);
-bindings.put("depth", depth);
+bindings.put("dataSetDepth", depth);
+bindings.put("processDepth", depth - 1);
 
 if (depth < 1) {
 ret = isDataSet ? 
gremlinQueryProvider.getQuery(FULL_LINEAGE_DATASET) :

http://git-wip-us.apache.org/repos/asf/atlas/blob/da78c483/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
--
diff --git 
a/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
 
b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
index 6d3b1a8..20c570f 100644
--- 
a/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
+++ 
b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
@@ -48,11 +48,11 @@ public class AtlasGremlin3QueryProvider extends 
AtlasGremlin2QueryProvider {
 case FULL_LINEAGE_DATASET:
 return "g.V().has('__guid', 
guid).repeat(__.inE(incomingEdgeLabel).as('e1').outV().outE(outgoingEdgeLabel).as('e2').inV()).emit().select('e1',
 'e2').toList()";
 case PARTIAL_LINEAGE_DATASET:
-return "g.V().has('__guid', 
guid).repeat(__.inE(incomingEdgeLabel).as('e1').outV().outE(outgoingEdgeLabel).as('e2').inV()).times(depth).emit().select('e1',
 'e2').toList()";
+return "g.V().has('__guid', 
guid).repeat(__.inE(incomingEdgeLabel).as('e1').outV().outE(outgoingEdgeLabel).as('e2').inV()).times(dataSetDepth).emit().select('e1',
 'e2').toList()";
 case FULL_LINEAGE_PROCESS:
 return "g.V().has('__guid', 
guid).outE(outgoingEdgeLabel).store('e').inV().repeat(__.inE(incomingEdgeLabel).store('e').outV().outE(outgoingEdgeLabel).store('e').inV()).cap('e').unfold().toList()";
 case PARTIAL_LINEAGE_PROCESS:
-return "g.V().has('__guid', 
guid).outE(outgoingEdgeLabel).store('e').inV().repeat(__.inE(incomingEdgeLabel).store('e').outV().outE(outgoingEdgeLabel).store('e').inV()).times(depth).cap('e').unfold().toList()";
+return "g.V().has('__guid', 
guid).outE(outgoingEdgeLabel).store('e').inV().until(loops().is(eq(processDepth))).repeat(__.inE(incomingEdgeLabel).store('e').outV().outE(outgoingEdgeLabel).store('e').inV()).cap('e').unfold().toList()";
 case TO_RANGE_LIST:
 return ".range(startIdx, endIdx).toList()";
 case RELATIONSHIP_SEARCH:

svn commit: r30433 - in /dev/atlas/0.8.3-rc1: ./ apache-atlas-0.8.3-sources.tar.gz apache-atlas-0.8.3-sources.tar.gz.asc apache-atlas-0.8.3-sources.tar.gz.md5 apache-atlas-0.8.3-sources.tar.gz.sha512

2018-10-26 Thread amestry 
Author: amestry
Date: Fri Oct 26 22:42:31 2018
New Revision: 30433

Log:
added 0.8.3-rc1 artifacts

Added:
dev/atlas/0.8.3-rc1/
dev/atlas/0.8.3-rc1/apache-atlas-0.8.3-sources.tar.gz   (with props)
dev/atlas/0.8.3-rc1/apache-atlas-0.8.3-sources.tar.gz.asc
dev/atlas/0.8.3-rc1/apache-atlas-0.8.3-sources.tar.gz.md5
dev/atlas/0.8.3-rc1/apache-atlas-0.8.3-sources.tar.gz.sha512

Added: dev/atlas/0.8.3-rc1/apache-atlas-0.8.3-sources.tar.gz
==
Binary file - no diff available.

Propchange: dev/atlas/0.8.3-rc1/apache-atlas-0.8.3-sources.tar.gz
--
svn:mime-type = application/octet-stream

Added: dev/atlas/0.8.3-rc1/apache-atlas-0.8.3-sources.tar.gz.asc
==
--- dev/atlas/0.8.3-rc1/apache-atlas-0.8.3-sources.tar.gz.asc (added)
+++ dev/atlas/0.8.3-rc1/apache-atlas-0.8.3-sources.tar.gz.asc Fri Oct 26 
22:42:31 2018
@@ -0,0 +1,16 @@
+-BEGIN PGP SIGNATURE-
+
+iQIzBAABCAAdFiEE21rYRZ40YST4Uh/KotzXCwzuIO4FAlvTl3oACgkQotzXCwzu
+IO6CSA//XrA2GVJFa4En1rN3XbE8uUtaINAEt9n9vMRcyGjTbq+ilmbuikqI7Erk
+gXoRNBSyqIEr34R3JVRVk771PpyFtwmuvfCzwls7CUxbflJlu5YFYyYo4VD7h6wW
+0semgKz3iWUGS+0uCjZpgBj1p9YJGnbhnByIS8xmMOTrwKCY1LFC31vqC59S91Fj
+G5ojeDz3V7tfd6t8SgWQRUF1vfTLFBTRch2S/xZv9dq67zP75DEJSZm8OiT9B4iP
+ED3BecMqa9XMIi/q4IbmI4+degJY2wRBBl8EQt/rta0jZF63DAhZsbhENqg/FOJv
+ah9xU8G55Da3BbZzJDL28Agw9r7jxe8YZi/DazFQMTTQrtJBEsoyXE9tSzwjaOH/
+dpZagVsQc6Vj2Ccvy6sQ7PZP93WO4bwTqtap8O30sEMhzXdOj/24xnMIsfmd9elr
+SkbY/QEtpelyly65XBuSPjmQflfy8mh7+lEcCSzNYAvkx3jKmf84pFmTaxIcC5lR
+keSPV92cusw/aBk7krl5dBmri/8GAIb+1ECBeLvRNTSf+dG+exW230qQYTReUnjr
+JtSXgRmdnSjy0fcjwjKKQAqdccRJPY/7rUtzscEHikyKOf2/ujhvJeD+9w7SWJkt
+3Znohb8oRXxBjmNgex0Bl8W4wDxbAkom+jleOVRsBHNtDCkoUYU=
+=XiBL
+-END PGP SIGNATURE-

Added: dev/atlas/0.8.3-rc1/apache-atlas-0.8.3-sources.tar.gz.md5
==
--- dev/atlas/0.8.3-rc1/apache-atlas-0.8.3-sources.tar.gz.md5 (added)
+++ dev/atlas/0.8.3-rc1/apache-atlas-0.8.3-sources.tar.gz.md5 Fri Oct 26 
22:42:31 2018
@@ -0,0 +1,2 @@
+apache-atlas-0.8.3-sources.tar.gz: 23 52 43 C5 CB 02 5E 07  CE 3A 83 65 C8 68 
95
+   7C

Added: dev/atlas/0.8.3-rc1/apache-atlas-0.8.3-sources.tar.gz.sha512
==
--- dev/atlas/0.8.3-rc1/apache-atlas-0.8.3-sources.tar.gz.sha512 (added)
+++ dev/atlas/0.8.3-rc1/apache-atlas-0.8.3-sources.tar.gz.sha512 Fri Oct 26 
22:42:31 2018
@@ -0,0 +1,4 @@
+apache-atlas-0.8.3-sources.tar.gz: BEFC7F7B E198A1B5 E120BF35 DE577636 B88E2469
+   F60A831B C4F7C804 C1483E50 4DC2F89C 4F9A7606
+   F2719FF8 41E39C0A C89050DF B95FC270 64B2FEEA
+   C1A214D1

[atlas] Git Push Summary

2018-10-26 Thread amestry 
Repository: atlas
Updated Tags:  refs/tags/release-0.8.3-rc1 [created] 1dfa5daaf

atlas git commit: ATLAS-2900: Export connected addressed case where imported file is not usable.

2018-10-26 Thread amestry 
Repository: atlas
Updated Branches:
  refs/heads/master 0cf62fd80 -> 572c5d648


ATLAS-2900: Export connected addressed case where imported file is not usable.

Signed-off-by: Ashutosh Mestry 


Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/572c5d64
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/572c5d64
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/572c5d64

Branch: refs/heads/master
Commit: 572c5d6480ed4e4d3b8da4084798c7740acd0ae3
Parents: 0cf62fd
Author: Ashutosh Mestry 
Authored: Thu Oct 25 21:59:31 2018 -0700
Committer: Ashutosh Mestry 
Committed: Fri Oct 26 13:45:52 2018 -0700

--
 .../atlas/repository/impexp/ExportService.java  | 27 
 .../atlas/repository/util/UniqueList.java   |  5 ++--
 .../impexp/ExportIncrementalTest.java   | 23 +
 .../repository/impexp/ExportServiceTest.java|  2 +-
 .../stocksDB-Entities/export-connected.json | 10 
 5 files changed, 53 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/atlas/blob/572c5d64/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
--
diff --git 
a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
 
b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
index 5e972a2..3558d2a 100644
--- 
a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
+++ 
b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
@@ -19,8 +19,6 @@ package org.apache.atlas.repository.impexp;
 
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.atlas.AtlasErrorCode;
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.AtlasServiceException;
 import org.apache.atlas.RequestContext;
 import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.TypeCategory;
@@ -135,18 +133,16 @@ public class ExportService {
 long startTime, long endTime) 
throws AtlasBaseException {
 int duration = getOperationDuration(startTime, endTime);
 
context.result.setSourceClusterName(AuditsWriter.getCurrentClusterName());
-
context.result.getData().getEntityCreationOrder().addAll(context.lineageProcessed);
-
context.sink.setExportOrder(context.result.getData().getEntityCreationOrder());
+context.addToEntityCreationOrder(context.lineageProcessed);
+
+context.sink.setExportOrder(context.entityCreationOrder.getList());
 context.sink.setTypesDef(context.result.getData().getTypesDef());
 context.result.setOperationStatus(getOverallOperationStatus(statuses));
 context.result.incrementMeticsCounter("duration", duration);
-auditsWriter.write(userName, context.result, startTime, endTime, 
context.result.getData().getEntityCreationOrder());
-clearContextData(context);
-context.sink.setResult(context.result);
-}
+auditsWriter.write(userName, context.result, startTime, endTime, 
context.entityCreationOrder.getList());
 
-private void clearContextData(ExportContext context) {
 context.result.setData(null);
+context.sink.setResult(context.result);
 }
 
 private int getOperationDuration(long startTime, long endTime) {
@@ -375,7 +371,7 @@ public class ExportService {
TraversalDirection direction) throws 
AtlasBaseException {
 
 if (!context.lineageProcessed.contains(guid) && 
context.doesTimestampQualify(entityWithExtInfo.getEntity())) {
-
context.result.getData().getEntityCreationOrder().add(entityWithExtInfo.getEntity().getGuid());
+
context.addToEntityCreationOrder(entityWithExtInfo.getEntity().getGuid());
 }
 
 addEntity(entityWithExtInfo, context);
@@ -546,7 +542,7 @@ public class ExportService {
 } else {
 List entities = 
context.getEntitiesWithModifiedTimestamp(entityWithExtInfo);
 for (AtlasEntity e : entities) {
-
context.result.getData().getEntityCreationOrder().add(e.getGuid());
+context.addToEntityCreationOrder(e.getGuid());
 context.addToSink(new AtlasEntityWithExtInfo(e));
 
context.result.incrementMeticsCounter(String.format("entity:%s", 
e.getTypeName()));
 }
@@ -718,6 +714,7 @@ public class ExportService {
 private static final String ATLAS_TYPE_HIVE_DB = "hive_db";
 
 
+final UniqueList  entityCreationOrder = new 
UniqueList<>();
 final Set guidsProcessed = new HashSet<>();
 final private UniqueList