incubator-atlas git commit: ATLAS-1961: Basic search improvement in use of Solr index for attribute filtering (# 5)

2017-07-20 Thread madhan 
Repository: incubator-atlas
Updated Branches:
  refs/heads/0.8-incubating 3197659dd -> 430f98101


ATLAS-1961: Basic search improvement in use of Solr index for attribute 
filtering (# 5)

(cherry picked from commit 62d85a4caebeb8f459c0a1774fade83cd49b9c35)


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/430f9810
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/430f9810
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/430f9810

Branch: refs/heads/0.8-incubating
Commit: 430f981019e48d27ab435f3acddf4396f40f65b3
Parents: 3197659
Author: Madhan Neethiraj 
Authored: Thu Jul 20 15:42:46 2017 -0700
Committer: Madhan Neethiraj 
Committed: Thu Jul 20 21:28:42 2017 -0700

--
 .../atlas/discovery/EntitySearchProcessor.java  |  2 --
 .../discovery/FullTextSearchProcessor.java  | 12 --
 .../apache/atlas/discovery/SearchProcessor.java | 23 +---
 3 files changed, 19 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/430f9810/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
--
diff --git 
a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
 
b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
index a3525c9..efcfb7f 100644
--- 
a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
+++ 
b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -18,10 +18,8 @@
 package org.apache.atlas.discovery;
 
 import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
-import org.apache.atlas.model.instance.AtlasEntity;
 import org.apache.atlas.repository.Constants;
 import org.apache.atlas.repository.graphdb.*;
-import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
 import org.apache.atlas.type.AtlasClassificationType;
 import org.apache.atlas.type.AtlasEntityType;
 import org.apache.atlas.utils.AtlasPerfTracer;

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/430f9810/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
--
diff --git 
a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
 
b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
index 1b19a0e..0d1c39b 100644
--- 
a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
+++ 
b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
@@ -20,12 +20,10 @@ package org.apache.atlas.discovery;
 import org.apache.atlas.model.discovery.SearchParameters;
 import org.apache.atlas.model.instance.AtlasEntity;
 import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.graph.GraphHelper;
 import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
 import org.apache.atlas.repository.graphdb.AtlasVertex;
 import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
 import org.apache.atlas.utils.AtlasPerfTracer;
-import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -55,7 +53,8 @@ public class FullTextSearchProcessor extends SearchProcessor {
 Set typeAndSubTypeNames = 
context.getEntityType().getTypeAndAllSubTypes();
 
 if (typeAndSubTypeNames.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY) 
{
-
queryString.append(AND_STR).append("(").append(StringUtils.join(typeAndSubTypeNames,
 SPACE_STRING)).append(")");
+queryString.append(AND_STR);
+appendIndexQueryValue(typeAndSubTypeNames, queryString);
 } else {
 LOG.warn("'{}' has too many subtypes ({}) to include in 
index-query; might cause poor performance",
  context.getEntityType().getTypeName(), 
typeAndSubTypeNames.size());
@@ -68,17 +67,14 @@ public class FullTextSearchProcessor extends 
SearchProcessor {
 Set typeAndSubTypeNames = 
context.getClassificationType().getTypeAndAllSubTypes();
 
 if (typeAndSubTypeNames.size() <= 
MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY) {
-
queryString.append(AND_STR).append("(").append(StringUtils.join(typeAndSubTypeNames,
 SPACE_STRING)).append(")");
+queryString.append(AND_STR);
+appendIndexQueryValue(typeAndSubTypeNames, queryString);
 } else {
 LOG.warn("'{}' has too many subtypes ({}) to include in 
index-query; might cause poor performance",
 context.getClassificationType().getTypeName(), 
typeAndSub

incubator-atlas git commit: ATLAS-1961: Basic search improvement in use of Solr index for attribute filtering (# 5)

2017-07-20 Thread madhan 
Repository: incubator-atlas
Updated Branches:
  refs/heads/master 13ba156f1 -> 62d85a4ca


ATLAS-1961: Basic search improvement in use of Solr index for attribute 
filtering (# 5)


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/62d85a4c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/62d85a4c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/62d85a4c

Branch: refs/heads/master
Commit: 62d85a4caebeb8f459c0a1774fade83cd49b9c35
Parents: 13ba156
Author: Madhan Neethiraj 
Authored: Thu Jul 20 15:42:46 2017 -0700
Committer: Madhan Neethiraj 
Committed: Thu Jul 20 21:28:13 2017 -0700

--
 .../atlas/discovery/EntitySearchProcessor.java  |  2 --
 .../discovery/FullTextSearchProcessor.java  | 12 --
 .../apache/atlas/discovery/SearchProcessor.java | 23 +---
 3 files changed, 19 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/62d85a4c/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
--
diff --git 
a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
 
b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
index a3525c9..efcfb7f 100644
--- 
a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
+++ 
b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -18,10 +18,8 @@
 package org.apache.atlas.discovery;
 
 import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
-import org.apache.atlas.model.instance.AtlasEntity;
 import org.apache.atlas.repository.Constants;
 import org.apache.atlas.repository.graphdb.*;
-import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
 import org.apache.atlas.type.AtlasClassificationType;
 import org.apache.atlas.type.AtlasEntityType;
 import org.apache.atlas.utils.AtlasPerfTracer;

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/62d85a4c/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
--
diff --git 
a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
 
b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
index 1b19a0e..0d1c39b 100644
--- 
a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
+++ 
b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
@@ -20,12 +20,10 @@ package org.apache.atlas.discovery;
 import org.apache.atlas.model.discovery.SearchParameters;
 import org.apache.atlas.model.instance.AtlasEntity;
 import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.graph.GraphHelper;
 import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
 import org.apache.atlas.repository.graphdb.AtlasVertex;
 import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
 import org.apache.atlas.utils.AtlasPerfTracer;
-import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -55,7 +53,8 @@ public class FullTextSearchProcessor extends SearchProcessor {
 Set typeAndSubTypeNames = 
context.getEntityType().getTypeAndAllSubTypes();
 
 if (typeAndSubTypeNames.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY) 
{
-
queryString.append(AND_STR).append("(").append(StringUtils.join(typeAndSubTypeNames,
 SPACE_STRING)).append(")");
+queryString.append(AND_STR);
+appendIndexQueryValue(typeAndSubTypeNames, queryString);
 } else {
 LOG.warn("'{}' has too many subtypes ({}) to include in 
index-query; might cause poor performance",
  context.getEntityType().getTypeName(), 
typeAndSubTypeNames.size());
@@ -68,17 +67,14 @@ public class FullTextSearchProcessor extends 
SearchProcessor {
 Set typeAndSubTypeNames = 
context.getClassificationType().getTypeAndAllSubTypes();
 
 if (typeAndSubTypeNames.size() <= 
MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY) {
-
queryString.append(AND_STR).append("(").append(StringUtils.join(typeAndSubTypeNames,
 SPACE_STRING)).append(")");
+queryString.append(AND_STR);
+appendIndexQueryValue(typeAndSubTypeNames, queryString);
 } else {
 LOG.warn("'{}' has too many subtypes ({}) to include in 
index-query; might cause poor performance",
 context.getClassificationType().getTypeName(), 
typeAndSubTypeNames.size());
 }
 }
 
-if (context.getSearchParameter

incubator-atlas git commit: ATLAS-1979: Update storm model relationship category and fix for UT and Coverity scan issues

2017-07-20 Thread madhan 
Repository: incubator-atlas
Updated Branches:
  refs/heads/master e0fb7dc17 -> 13ba156f1


ATLAS-1979: Update storm model relationship category and fix for UT and 
Coverity scan issues

Signed-off-by: Madhan Neethiraj 


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/13ba156f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/13ba156f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/13ba156f

Branch: refs/heads/master
Commit: 13ba156f1d76caaa969e2f6fd93e2387307ae7c7
Parents: e0fb7dc
Author: Sarath Subramanian 
Authored: Thu Jul 20 14:12:54 2017 -0700
Committer: Madhan Neethiraj 
Committed: Thu Jul 20 19:03:40 2017 -0700

--
 addons/models/0080-storm_model.json   |  2 +-
 .../repository/store/graph/v1/EntityGraphMapper.java  |  9 +++--
 .../graph/ReverseReferenceUpdateSoftDeleteTest.java   |  1 +
 .../store/graph/v1/AtlasRelationshipStoreV1Test.java  | 10 +++---
 4 files changed, 16 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/13ba156f/addons/models/0080-storm_model.json
--
diff --git a/addons/models/0080-storm_model.json 
b/addons/models/0080-storm_model.json
index b008c7a..ab3d603 100644
--- a/addons/models/0080-storm_model.json
+++ b/addons/models/0080-storm_model.json
@@ -147,7 +147,7 @@
 {
 "name": "storm_topology_nodes",
 "typeVersion": "1.0",
-"relationshipCategory": "ASSOCIATION",
+"relationshipCategory": "AGGREGATION",
 "endDef1": {
 "type": "storm_topology",
 "name": "nodes",

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/13ba156f/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
--
diff --git 
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
 
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
index 1282be5..b8fd70e 100644
--- 
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
+++ 
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
@@ -1075,7 +1075,7 @@ public class EntityGraphMapper {
 StringnewEntityId = getIdFromVertex(newEntityVertex);
 AtlasEdge ret = currentEdge;
 
-if (!currentEntityId.equals(newEntityId) && newEntityVertex != null) {
+if (!currentEntityId.equals(newEntityId)) {
 // create a new relationship edge to the new attribute vertex from 
the instance
 String relationshipName = 
AtlasGraphUtilsV1.getTypeName(currentEdge);
 
@@ -1230,6 +1230,11 @@ public class EntityGraphMapper {
 // get the classification vertex from entity
 String  relationshipLabel= 
GraphHelper.getTraitLabel(entityTypeName, classification.getTypeName());
 AtlasEdge   classificationEdge   = 
graphHelper.getEdgeForLabel(instanceVertex, relationshipLabel);
+
+if (classificationEdge == null) {
+throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, 
"classificationEdge is null for label: " + relationshipLabel);
+}
+
 AtlasVertex classificationVertex = classificationEdge.getInVertex();
 
 if (LOG.isDebugEnabled()) {
@@ -1363,7 +1368,7 @@ public class EntityGraphMapper {
 private boolean objectIdsContain(Collection objectIds, 
AtlasObjectId objectId) {
 boolean ret = false;
 
-if (objectIds != null && objectIds.isEmpty()) {
+if (CollectionUtils.isEmpty(objectIds)) {
 ret = false;
 
 } else {

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/13ba156f/repository/src/test/java/org/apache/atlas/repository/graph/ReverseReferenceUpdateSoftDeleteTest.java
--
diff --git 
a/repository/src/test/java/org/apache/atlas/repository/graph/ReverseReferenceUpdateSoftDeleteTest.java
 
b/repository/src/test/java/org/apache/atlas/repository/graph/ReverseReferenceUpdateSoftDeleteTest.java
index c5eda37..ce43bdc 100644
--- 
a/repository/src/test/java/org/apache/atlas/repository/graph/ReverseReferenceUpdateSoftDeleteTest.java
+++ 
b/repository/src/test/java/org/apache/atlas/repository/graph/ReverseReferenceUpdateSoftDeleteTest.java
@@ -48,6 +48,7 @@ public class ReverseReferenceUpdateSoftDeleteTest extends 
ReverseReferenceUpdate
 AtlasVertex vertexForGUID = 
GraphHelper.getInstance().getVertexForGUID(referencingInstance.getId()._getId());
 String edgeLabel =

incubator-atlas git commit: ATLAS-1961: Basic search improvement in use of Solr index for attribute filtering (# 4)

2017-07-20 Thread madhan 
Repository: incubator-atlas
Updated Branches:
  refs/heads/0.8-incubating f5cee9e38 -> 3197659dd


ATLAS-1961: Basic search improvement in use of Solr index for attribute 
filtering (# 4)

(cherry picked from commit e0fb7dc17000724fada89cb573db9b4fa1654a2e)


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/3197659d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/3197659d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/3197659d

Branch: refs/heads/0.8-incubating
Commit: 3197659dddfc7d1ada0715e1fac5b7afa81a6df6
Parents: f5cee9e
Author: Madhan Neethiraj 
Authored: Thu Jul 20 01:19:57 2017 -0700
Committer: Madhan Neethiraj 
Committed: Thu Jul 20 14:30:34 2017 -0700

--
 .../ClassificationSearchProcessor.java  | 35 +---
 .../atlas/discovery/EntitySearchProcessor.java  | 26 ---
 .../discovery/FullTextSearchProcessor.java  | 27 ---
 .../apache/atlas/discovery/SearchProcessor.java | 18 +-
 4 files changed, 68 insertions(+), 38 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3197659d/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
--
diff --git 
a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
 
b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
index b6e0de5..745f9d7 100644
--- 
a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
+++ 
b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
@@ -58,7 +58,7 @@ public class ClassificationSearchProcessor extends 
SearchProcessor {
 if (useSolrSearch) {
 StringBuilder solrQuery = new StringBuilder();
 
-constructTypeTestQuery(solrQuery, classificationType, 
typeAndSubTypes);
+constructTypeTestQuery(solrQuery, typeAndSubTypes);
 constructFilterQuery(solrQuery, classificationType, 
filterCriteria, solrAttributes);
 
 String solrQueryString = 
STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")");
@@ -95,16 +95,22 @@ public class ClassificationSearchProcessor extends 
SearchProcessor {
 }
 
 try {
-final int startIdx  = context.getSearchParameters().getOffset();
-final int limit = context.getSearchParameters().getLimit();
-int   qryOffset = nextProcessor == null ? startIdx : 0;
-int   resultIdx = qryOffset;
+final int startIdx   = 
context.getSearchParameters().getOffset();
+final int limit  = 
context.getSearchParameters().getLimit();
+final boolean activeOnly = 
context.getSearchParameters().getExcludeDeletedEntities();
+
+// query to start at 0, even though startIdx can be higher - 
because few results in earlier retrieval could
+// have been dropped: like non-active-entities or 
duplicate-entities (same entity pointed to by multiple
+// classifications in the result)
+//
+// first 'startIdx' number of entries will be ignored
+int qryOffset = 0;
+int resultIdx = qryOffset;
 
 final Set   processedGuids = new HashSet<>();
 final List entityVertices = new ArrayList<>();
 final List classificationVertices = new ArrayList<>();
 
-
 for (; ret.size() < limit; qryOffset += limit) {
 entityVertices.clear();
 classificationVertices.clear();
@@ -138,15 +144,20 @@ public class ClassificationSearchProcessor extends 
SearchProcessor {
 
 for (AtlasEdge edge : edges) {
 AtlasVertex entityVertex = edge.getOutVertex();
-String  guid = 
AtlasGraphUtilsV1.getIdFromVertex(entityVertex);
 
-if (!processedGuids.contains(guid)) {
-if 
(!context.getSearchParameters().getExcludeDeletedEntities() || 
AtlasGraphUtilsV1.getState(entityVertex) == AtlasEntity.Status.ACTIVE) {
-entityVertices.add(entityVertex);
-}
+if (activeOnly && 
AtlasGraphUtilsV1.getState(entityVertex) != AtlasEntity.Status.ACTIVE) {
+continue;
+}
+
+String guid = 
AtlasGraphUtilsV1.getIdFromVertex(entityVertex);
 
-processedGuids.add(guid);
+if (processedGuids.contains(guid)) {
+continue;

incubator-atlas git commit: ATLAS-1961: Basic search improvement in use of Solr index for attribute filtering (# 4)

2017-07-20 Thread madhan 
Repository: incubator-atlas
Updated Branches:
  refs/heads/master 3d3be4084 -> e0fb7dc17


ATLAS-1961: Basic search improvement in use of Solr index for attribute 
filtering (# 4)


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/e0fb7dc1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/e0fb7dc1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/e0fb7dc1

Branch: refs/heads/master
Commit: e0fb7dc17000724fada89cb573db9b4fa1654a2e
Parents: 3d3be40
Author: Madhan Neethiraj 
Authored: Thu Jul 20 01:19:57 2017 -0700
Committer: Madhan Neethiraj 
Committed: Thu Jul 20 14:23:18 2017 -0700

--
 .../ClassificationSearchProcessor.java  | 35 +---
 .../atlas/discovery/EntitySearchProcessor.java  | 26 ---
 .../discovery/FullTextSearchProcessor.java  | 27 ---
 .../apache/atlas/discovery/SearchProcessor.java | 18 +-
 4 files changed, 68 insertions(+), 38 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e0fb7dc1/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
--
diff --git 
a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
 
b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
index b6e0de5..745f9d7 100644
--- 
a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
+++ 
b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
@@ -58,7 +58,7 @@ public class ClassificationSearchProcessor extends 
SearchProcessor {
 if (useSolrSearch) {
 StringBuilder solrQuery = new StringBuilder();
 
-constructTypeTestQuery(solrQuery, classificationType, 
typeAndSubTypes);
+constructTypeTestQuery(solrQuery, typeAndSubTypes);
 constructFilterQuery(solrQuery, classificationType, 
filterCriteria, solrAttributes);
 
 String solrQueryString = 
STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")");
@@ -95,16 +95,22 @@ public class ClassificationSearchProcessor extends 
SearchProcessor {
 }
 
 try {
-final int startIdx  = context.getSearchParameters().getOffset();
-final int limit = context.getSearchParameters().getLimit();
-int   qryOffset = nextProcessor == null ? startIdx : 0;
-int   resultIdx = qryOffset;
+final int startIdx   = 
context.getSearchParameters().getOffset();
+final int limit  = 
context.getSearchParameters().getLimit();
+final boolean activeOnly = 
context.getSearchParameters().getExcludeDeletedEntities();
+
+// query to start at 0, even though startIdx can be higher - 
because few results in earlier retrieval could
+// have been dropped: like non-active-entities or 
duplicate-entities (same entity pointed to by multiple
+// classifications in the result)
+//
+// first 'startIdx' number of entries will be ignored
+int qryOffset = 0;
+int resultIdx = qryOffset;
 
 final Set   processedGuids = new HashSet<>();
 final List entityVertices = new ArrayList<>();
 final List classificationVertices = new ArrayList<>();
 
-
 for (; ret.size() < limit; qryOffset += limit) {
 entityVertices.clear();
 classificationVertices.clear();
@@ -138,15 +144,20 @@ public class ClassificationSearchProcessor extends 
SearchProcessor {
 
 for (AtlasEdge edge : edges) {
 AtlasVertex entityVertex = edge.getOutVertex();
-String  guid = 
AtlasGraphUtilsV1.getIdFromVertex(entityVertex);
 
-if (!processedGuids.contains(guid)) {
-if 
(!context.getSearchParameters().getExcludeDeletedEntities() || 
AtlasGraphUtilsV1.getState(entityVertex) == AtlasEntity.Status.ACTIVE) {
-entityVertices.add(entityVertex);
-}
+if (activeOnly && 
AtlasGraphUtilsV1.getState(entityVertex) != AtlasEntity.Status.ACTIVE) {
+continue;
+}
+
+String guid = 
AtlasGraphUtilsV1.getIdFromVertex(entityVertex);
 
-processedGuids.add(guid);
+if (processedGuids.contains(guid)) {
+continue;
 }
+
+entityVertices.add(entityVertex);
+
+

[2/2] incubator-atlas git commit: Fix ODF build issues introduced by package rename

2017-07-20 Thread madhan 
Fix ODF build issues introduced by package rename

(cherry picked from commit 21925d18fd9527e16a7a7a19ae2ec5f1da4c6e92)
Signed-off-by: Madhan Neethiraj 


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/cac4bee5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/cac4bee5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/cac4bee5

Branch: refs/heads/feature-odf
Commit: cac4bee572845f2fa0859cf7a5a14b8536d7a618
Parents: 14e70b6
Author: Robert Kern 
Authored: Mon Jul 3 18:11:06 2017 +0200
Committer: Madhan Neethiraj 
Committed: Thu Jul 20 12:48:44 2017 -0700

--
 .../src/test/resources/projects/it1/goal.txt| 13 -
 .../atlas/odf/odf-implementation.properties |  2 +-
 .../atlas/odf/core/ODFInternalFactory.java  |  2 +-
 .../core/store/DefaultConfigurationStorage.java | 50 
 .../odf-default-implementation.properties   | 32 +++--
 .../internal/odf-initial-configuration.json |  2 +-
 .../org/apache/atlas/odf/core/odfversion.txt| 15 +-
 .../test/configuration/ValidationTests.java |  2 +-
 .../DeclarativeRequestMapperTest.java   |  2 +-
 .../DiscoveryServiceManagerTest.java|  4 +-
 .../resources/META-INF/odf/odf-runtimes.txt |  2 +-
 .../internal/odf-initial-configuration.json | 14 +++---
 .../odf/core/test/messaging/kafka/tracker1.json |  2 +-
 .../atlas/odf/odf-implementation.properties |  8 ++--
 .../atlas/odf/odf-implementation.properties |  2 +-
 .../atlas/odf/odf-implementation.properties |  6 +--
 odf/odf-spark-example-application/pom.xml   |  2 +-
 .../atlas/odf/odf-implementation.properties |  2 +-
 .../atlas/odf/odf-implementation.properties |  2 +-
 .../atlas/odf/odf-implementation.properties |  2 +-
 odf/odf-web/.gitignore  |  1 +
 odf/odf-web/package.json|  9 ++--
 odf/odf-web/pom.xml |  4 +-
 23 files changed, 104 insertions(+), 76 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/cac4bee5/odf/odf-archetype-discoveryservice/src/test/resources/projects/it1/goal.txt
--
diff --git 
a/odf/odf-archetype-discoveryservice/src/test/resources/projects/it1/goal.txt 
b/odf/odf-archetype-discoveryservice/src/test/resources/projects/it1/goal.txt
index 3cb5141..31ed2f8 100755
--- 
a/odf/odf-archetype-discoveryservice/src/test/resources/projects/it1/goal.txt
+++ 
b/odf/odf-archetype-discoveryservice/src/test/resources/projects/it1/goal.txt
@@ -1,14 +1 @@
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
 clean verify

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/cac4bee5/odf/odf-atlas/src/main/resources/org/apache/atlas/odf/odf-implementation.properties
--
diff --git 
a/odf/odf-atlas/src/main/resources/org/apache/atlas/odf/odf-implementation.properties
 
b/odf/odf-atlas/src/main/resources/org/apache/atlas/odf/odf-implementation.properties
index 500aae7..8587cc8 100755
--- 
a/odf/odf-atlas/src/main/resources/org/apache/atlas/odf/odf-implementation.properties
+++ 
b/odf/odf-atlas/src/main/resources/org/apache/atlas/odf/odf-implementation.properties
@@ -12,4 +12,4 @@
 # limitations under the License.
 #
 # Overwrite default implementation
-MetadataStore=org.apache.atlas.odf.core.metadata.atlas.AtlasMetadataStore
+org.apache.atlas.odf.api.metadata.MetadataStore=org.apache.atlas.odf.core.metadata.atlas.AtlasMetadataStore

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/cac4bee5/odf/odf-core/src/main/java/org/apache/atlas/odf/core/ODFInternalFactory.java
--
diff --git 
a/odf/odf-core/src/main/java/org/apache/atlas/odf/core/ODFInternalFactory.java 
b/odf/odf-core/src/main/java/org/apache/atlas/odf/core/ODFInternalFactory.java
index 4fd09a7..9023b5d 100755
--- 
a/odf/odf-core/src/main/java/org/apache/atlas/odf/core/ODFInternalFactory.java
+++ 
b/odf/odf-core/src/main/java/org/apache/atlas/odf/core/ODFInternalFactory.java
@@ -34,7 +34,7 @@ public class ODFInternalFactory {
 
Logger logger = 
Logger.getLo

[1/2] incubator-atlas git commit: Replace misleading license headers in markdown files

2017-07-20 Thread madhan 
Repository: incubator-atlas
Updated Branches:
  refs/heads/feature-odf f24d94fad -> cac4bee57


Replace misleading license headers in markdown files

(cherry picked from commit d74f50a94d13c527f0227f68227bbda1b01dd67e)
Signed-off-by: Madhan Neethiraj 


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/14e70b6a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/14e70b6a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/14e70b6a

Branch: refs/heads/feature-odf
Commit: 14e70b6a1c319821f0d06f948eed373a3c387243
Parents: f24d94f
Author: Robert Kern 
Authored: Mon Jul 3 19:33:19 2017 +0200
Committer: Madhan Neethiraj 
Committed: Thu Jul 20 12:48:34 2017 -0700

--
 odf/odf-doc/src/site/markdown/api-reference.md  | 30 ---
 odf/odf-doc/src/site/markdown/build.md  | 40 -
 odf/odf-doc/src/site/markdown/configuration.md  | 30 ---
 odf/odf-doc/src/site/markdown/data-model.md | 30 ---
 .../site/markdown/discovery-service-tutorial.md | 30 ---
 .../src/site/markdown/discovery-services.md | 30 ---
 odf/odf-doc/src/site/markdown/examples.md   | 30 ---
 .../site/markdown/first-analysis-tutorial.md| 30 ---
 odf/odf-doc/src/site/markdown/first-steps.md| 30 ---
 odf/odf-doc/src/site/markdown/index.md  | 31 ---
 odf/odf-doc/src/site/markdown/install.md| 30 ---
 odf/odf-doc/src/site/markdown/jenkins-build.md  | 93 
 .../src/site/markdown/odf-metadata-api.md   | 30 ---
 odf/odf-doc/src/site/markdown/operations.md | 30 ---
 .../spark-discovery-service-tutorial.md | 30 ---
 odf/odf-doc/src/site/markdown/test-env.md   | 30 ---
 .../src/site/markdown/troubleshooting.md| 30 ---
 odf/odf-doc/src/site/site.xml   |  3 -
 18 files changed, 273 insertions(+), 314 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/14e70b6a/odf/odf-doc/src/site/markdown/api-reference.md
--
diff --git a/odf/odf-doc/src/site/markdown/api-reference.md 
b/odf/odf-doc/src/site/markdown/api-reference.md
index f0bb24f..c03ada2 100755
--- a/odf/odf-doc/src/site/markdown/api-reference.md
+++ b/odf/odf-doc/src/site/markdown/api-reference.md
@@ -1,16 +1,20 @@
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
+
 
 # API reference
 

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/14e70b6a/odf/odf-doc/src/site/markdown/build.md
--
diff --git a/odf/odf-doc/src/site/markdown/build.md 
b/odf/odf-doc/src/site/markdown/build.md
index 4a6d7ac..8571973 100755
--- a/odf/odf-doc/src/site/markdown/build.md
+++ b/odf/odf-doc/src/site/markdown/build.md
@@ -1,16 +1,20 @@
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
+
 
 # Build
 
@@ -31,15 +35,6 @@ If you run these commands and you see similar output you 
should be all set:
$ git --version
git version 2.7.4
 
-The build currently depends on the [IA data quality project][3] 
(https://github.com/YSAILLET/dataquality) to be available. This can be achieved 
in two ways:
-1. Make sure that you can reach the [IIS Maven Repository][1]. If this doesn't 
work, try to authenticate against the Littleton firewall, e.g., by opening the 
[RTC dashboard][2].
-2. Build the [IA data quality project][3] on your local machine, typically 
with these commands
-
-   git clone https://github.com/dataquality
-   cd dataquality
-   mvn clean install
-
-
 ### Additional Prerequisites on Windows
 
 - For the build: The directory C:\tmp needs to exist
@@ -110,4 +105,3 @@ maven provided by the m2e plugin. Instead you wi

incubator-atlas git commit: ATLAS-1968: updated import documentation for fileName option usage

2017-07-20 Thread madhan 
Repository: incubator-atlas
Updated Branches:
  refs/heads/0.8-incubating f528a64c3 -> f5cee9e38


ATLAS-1968: updated import documentation for fileName option usage

Signed-off-by: Madhan Neethiraj 
(cherry picked from commit 3d3be40848775b43c97f4f94df47591762e78d0b)


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/f5cee9e3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/f5cee9e3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/f5cee9e3

Branch: refs/heads/0.8-incubating
Commit: f5cee9e38c7c61a89208a223f4d98d2804e5fca9
Parents: f528a64
Author: ashutoshm 
Authored: Thu Jul 20 09:56:04 2017 -0700
Committer: Madhan Neethiraj 
Committed: Thu Jul 20 10:53:39 2017 -0700

--
 docs/src/site/twiki/Import-API-Options.twiki| 22 
 docs/src/site/twiki/Import-API.twiki|  6 +++---
 .../atlas/model/impexp/AtlasImportRequest.java  | 18 +---
 .../atlas/repository/impexp/ImportService.java  |  2 +-
 4 files changed, 41 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f5cee9e3/docs/src/site/twiki/Import-API-Options.twiki
--
diff --git a/docs/src/site/twiki/Import-API-Options.twiki 
b/docs/src/site/twiki/Import-API-Options.twiki
index 8588784..cc09fc7 100644
--- a/docs/src/site/twiki/Import-API-Options.twiki
+++ b/docs/src/site/twiki/Import-API-Options.twiki
@@ -111,3 +111,25 @@ To use the option, set the contents of 
_importOptions.json_ to:
   }
 }
 
+
+--- Specifying File to be Imported From Server Location
+
+In scenario where the file to be imported is present at a location on the 
server, the _importfile_ API can be used. It behaves like the Import API.
+
+To use the option, set the contents of _importOptions.json_ to:
+
+{
+
+  "options": {
+"fileName": "/root/fileToBeImported.zip"
+  }
+}
+
+
+_CURL_
+
+curl -g -X POST -u adminuser:password -H "Content-Type: application/json"
+-H "Cache-Control: no-cache"
+-d r...@importoptions.json
+"http://localhost:21000/api/atlas/admin/importfile";
+

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f5cee9e3/docs/src/site/twiki/Import-API.twiki
--
diff --git a/docs/src/site/twiki/Import-API.twiki 
b/docs/src/site/twiki/Import-API.twiki
index b5de113..00ac9c4 100644
--- a/docs/src/site/twiki/Import-API.twiki
+++ b/docs/src/site/twiki/Import-API.twiki
@@ -24,7 +24,7 @@ The general approach is:
 | _Description_|Provide the path of the file to be imported.|
 | _URL_ |_api/atlas/admin/importfile_ |
 | _Method_ |_POST_ |
-| _URL Parameters_ |_?FILENAME=_  Specify the options as 
name-value pairs. Use _FILENAME_ to specify the file path. |
+| _URL Parameters_ |_None_ |
 | _Data Parameters_|_None_|
 | _Success Response_ | _!AtlasImporResult_ is returned as JSON. See details 
below.|
 |_Error Response_|Errors that are handled within the system will be returned 
as _!AtlasBaseException_. |
@@ -68,8 +68,8 @@ curl -g -X POST -u adminuser:password -H "Content-Type: 
multipart/form-data"
 
 The call below performs Import of _!QuickStart_ database using a ZIP file 
available on server.
 
-curl -X POST -u adminuser:password -H "Cache-Control: no-cache"
-"http://localhost:21000/api/atlas/admin/importFile?FILENAME=/root/quickStartDB.zip";
 > quickStartDB-import-result.json
+curl -X POST -u adminuser:password -H "Cache-Control: no-cache" -d 
./importOptions.json
+"http://localhost:21000/api/atlas/admin/importFile"; > 
quickStartDB-import-result.json
 
 
 Below is the _!AtlasImportResult_ JSON for an import that contains _hive_db_.

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f5cee9e3/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
--
diff --git 
a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java 
b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
index 4d2ac62..304ec5d 100644
--- a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
+++ b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
@@ -36,9 +36,11 @@ import static 
org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONL
 @JsonIgnoreProperties(ignoreUnknown=true)
 public class AtlasImportRequest implements Serializable {
 private static final long   serialVersionUID = 1L;
-public  static final String TRANSFORMS_KEY   = "transforms";
-private static final String START_POSITION_KEY = "startPosition";
-private static final String START_GUID_KEY = "startGuid";
+
+public  static final

incubator-atlas git commit: ATLAS-1968: updated import documentation for fileName option usage

2017-07-20 Thread madhan 
Repository: incubator-atlas
Updated Branches:
  refs/heads/master 02e4e86b5 -> 3d3be4084


ATLAS-1968: updated import documentation for fileName option usage

Signed-off-by: Madhan Neethiraj 


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/3d3be408
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/3d3be408
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/3d3be408

Branch: refs/heads/master
Commit: 3d3be40848775b43c97f4f94df47591762e78d0b
Parents: 02e4e86
Author: ashutoshm 
Authored: Thu Jul 20 09:56:04 2017 -0700
Committer: Madhan Neethiraj 
Committed: Thu Jul 20 10:41:27 2017 -0700

--
 docs/src/site/twiki/Import-API-Options.twiki| 22 
 docs/src/site/twiki/Import-API.twiki|  6 +++---
 .../atlas/model/impexp/AtlasImportRequest.java  | 18 +---
 .../atlas/repository/impexp/ImportService.java  |  2 +-
 4 files changed, 41 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3d3be408/docs/src/site/twiki/Import-API-Options.twiki
--
diff --git a/docs/src/site/twiki/Import-API-Options.twiki 
b/docs/src/site/twiki/Import-API-Options.twiki
index 8588784..cc09fc7 100644
--- a/docs/src/site/twiki/Import-API-Options.twiki
+++ b/docs/src/site/twiki/Import-API-Options.twiki
@@ -111,3 +111,25 @@ To use the option, set the contents of 
_importOptions.json_ to:
   }
 }
 
+
+--- Specifying File to be Imported From Server Location
+
+In scenario where the file to be imported is present at a location on the 
server, the _importfile_ API can be used. It behaves like the Import API.
+
+To use the option, set the contents of _importOptions.json_ to:
+
+{
+
+  "options": {
+"fileName": "/root/fileToBeImported.zip"
+  }
+}
+
+
+_CURL_
+
+curl -g -X POST -u adminuser:password -H "Content-Type: application/json"
+-H "Cache-Control: no-cache"
+-d r...@importoptions.json
+"http://localhost:21000/api/atlas/admin/importfile";
+

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3d3be408/docs/src/site/twiki/Import-API.twiki
--
diff --git a/docs/src/site/twiki/Import-API.twiki 
b/docs/src/site/twiki/Import-API.twiki
index b5de113..00ac9c4 100644
--- a/docs/src/site/twiki/Import-API.twiki
+++ b/docs/src/site/twiki/Import-API.twiki
@@ -24,7 +24,7 @@ The general approach is:
 | _Description_|Provide the path of the file to be imported.|
 | _URL_ |_api/atlas/admin/importfile_ |
 | _Method_ |_POST_ |
-| _URL Parameters_ |_?FILENAME=_  Specify the options as 
name-value pairs. Use _FILENAME_ to specify the file path. |
+| _URL Parameters_ |_None_ |
 | _Data Parameters_|_None_|
 | _Success Response_ | _!AtlasImporResult_ is returned as JSON. See details 
below.|
 |_Error Response_|Errors that are handled within the system will be returned 
as _!AtlasBaseException_. |
@@ -68,8 +68,8 @@ curl -g -X POST -u adminuser:password -H "Content-Type: 
multipart/form-data"
 
 The call below performs Import of _!QuickStart_ database using a ZIP file 
available on server.
 
-curl -X POST -u adminuser:password -H "Cache-Control: no-cache"
-"http://localhost:21000/api/atlas/admin/importFile?FILENAME=/root/quickStartDB.zip";
 > quickStartDB-import-result.json
+curl -X POST -u adminuser:password -H "Cache-Control: no-cache" -d 
./importOptions.json
+"http://localhost:21000/api/atlas/admin/importFile"; > 
quickStartDB-import-result.json
 
 
 Below is the _!AtlasImportResult_ JSON for an import that contains _hive_db_.

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/3d3be408/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
--
diff --git 
a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java 
b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
index 4d2ac62..304ec5d 100644
--- a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
+++ b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
@@ -36,9 +36,11 @@ import static 
org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONL
 @JsonIgnoreProperties(ignoreUnknown=true)
 public class AtlasImportRequest implements Serializable {
 private static final long   serialVersionUID = 1L;
-public  static final String TRANSFORMS_KEY   = "transforms";
-private static final String START_POSITION_KEY = "startPosition";
-private static final String START_GUID_KEY = "startGuid";
+
+public  static final String TRANSFORMS_KEY = "transforms";
+private static final String S