This is an automated email from the ASF dual-hosted git repository.
pinal pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 9fb546708 ATLAS-4442: Support Relationship search
9fb546708 is described below
commit 9fb5467089d08bff16d3a3031178ade93b3a1852
Author: Pinal Shah <pinal.s...@freestoneinfotech.com>
AuthorDate: Tue Sep 27 16:59:39 2022 +0530
ATLAS-4442: Support Relationship search
Signed-off-by: Pinal Shah <pinal.s...@freestoneinfotech.com>
(cherry picked from commit 9e6fc31ce8450f784515e57c7df3d7fa57f216de)
---
.../org/apache/atlas/repository/Constants.java | 2 +
docs/src/documents/Search/SearchRelationship.md | 132 +++++++++++
.../images/twiki/relationship_search_model.png | Bin 0 -> 68064 bytes
.../atlas/repository/graphdb/AtlasIndexQuery.java | 35 +++
.../graphdb/janus/AtlasJanusIndexQuery.java | 85 ++++++-
.../atlas/model/discovery/AtlasSearchResult.java | 43 +++-
.../discovery/RelationshipSearchParameters.java | 111 +++++++++
.../atlas/model/discovery/SearchParameters.java | 23 ++
.../model/instance/AtlasRelationshipHeader.java | 8 +
.../atlas/model/profile/AtlasUserSavedSearch.java | 12 +-
.../atlas/discovery/AtlasDiscoveryService.java | 8 +
.../atlas/discovery/EntityDiscoveryService.java | 57 +++++
.../atlas/discovery/GraphIndexQueryBuilder.java | 11 +
.../discovery/RelationshipSearchProcessor.java | 210 +++++++++++++++++
.../org/apache/atlas/discovery/SearchContext.java | 56 +++++
.../apache/atlas/discovery/SearchProcessor.java | 62 ++++-
.../repository/graph/GraphBackedSearchIndexer.java | 51 +++-
.../repository/patches/AtlasPatchManager.java | 1 +
.../repository/patches/EdgePatchProcessor.java | 213 +++++++++++++++++
.../patches/RelationshipTypeNamePatch.java | 107 +++++++++
.../bootstrap/AtlasTypeDefStoreInitializer.java | 12 +
.../store/graph/v2/AtlasGraphUtilsV2.java | 7 +-
.../store/graph/v2/AtlasRelationshipStoreV2.java | 2 +
.../store/graph/v2/EntityGraphRetriever.java | 11 +
.../org/apache/atlas/util/SearchPredicateUtil.java | 31 +++
.../test/java/org/apache/atlas/BasicTestSetup.java | 187 ++++++++++++++-
.../atlas/discovery/AtlasDiscoveryServiceTest.java | 13 ++
.../discovery/RelationshipSearchProcessorTest.java | 200 ++++++++++++++++
.../test/resources/relationship_search_def.json | 259 +++++++++++++++++++++
.../resources/solr/core-template/solrconfig.xml | 2 +-
.../org/apache/atlas/web/rest/DiscoveryREST.java | 76 ++++++
31 files changed, 2011 insertions(+), 16 deletions(-)
diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java
b/common/src/main/java/org/apache/atlas/repository/Constants.java
index 7cd67a04c..c84e1b2d0 100644
--- a/common/src/main/java/org/apache/atlas/repository/Constants.java
+++ b/common/src/main/java/org/apache/atlas/repository/Constants.java
@@ -46,6 +46,8 @@ public final class Constants {
* Entity type name property key.
*/
public static final String ENTITY_TYPE_PROPERTY_KEY =
encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "typeName");
+ public static final String RELATIONSHIP_TYPE_PROPERTY_KEY =
encodePropertyKey(RELATIONSHIP_PROPERTY_KEY_PREFIX + ENTITY_TYPE_PROPERTY_KEY);
+
public static final String TYPE_NAME_INTERNAL =
INTERNAL_PROPERTY_KEY_PREFIX + "internal";
public static final String ASSET_ENTITY_TYPE = "Asset";
public static final String OWNER_ATTRIBUTE = "owner";
diff --git a/docs/src/documents/Search/SearchRelationship.md
b/docs/src/documents/Search/SearchRelationship.md
new file mode 100644
index 000000000..c9fd78340
--- /dev/null
+++ b/docs/src/documents/Search/SearchRelationship.md
@@ -0,0 +1,132 @@
+---
+name: Relationship Search
+route: /SearchRelationship
+menu: Documentation
+submenu: Search
+---
+
+import themen from 'theme/styles/styled-colors';
+import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
+import SyntaxHighlighter from 'react-syntax-highlighter';
+import Img from 'theme/components/shared/Img'
+
+# Relationship Search
+
+Apache Atlas is a metadata governance tool that features the ability to search
for pre-defined entities of the defined model.
+
+We are able to search for these entities by specifying generic attributes
(such as ‘name’, ‘qualified name’, ‘description’ etc), entity type definition
specific attributes (attributes unique to certain entities such as ‘cluster
name’ for hdfs_path type entities) and also on parameters like classifications
applied, sub-types of entities and deleted entities. However these results only
return entities which fulfil the parameters of the search queries.
+
+A relationship, which also follows a similar structure to an entity, describes
various metadata between two entity end-points. For example, let us assume that
in the case of a relationship between hive_table and hive_db, the relationship
type name is called hive_table_db and it can have its own metadata which can be
added as an attributes to this model.
+
+The Relationship search allows you to get the metadata between two entity
model, by querying using relationship type name, and it also has support for
filtering on the relationship attribute(s).
+
+The entire query structure can be represented using the following JSON
structure (called RelationshipSearchParameters)
+
+<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
+{`{
+ "relationshipName": "hive_table_db",
+ "excludeDeletedEntities": true,
+ "offset": 0,
+ "limit": 25,
+ "relationshipFilters": { },
+ "sortBy": "table_name",
+ "sortOrder": ASCENDING,
+ "marker": "*"
+}`}
+</SyntaxHighlighter>
+
+**Field description**
+
+ <SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
+ {`relationshipName: the type of relationship to look for
+excludeDeletedEntities: should the search exclude deleted entities? (default:
true)
+offset: starting offset of the result set (useful for
pagination)
+limit: max number of results to fetch
+relationshipFilters: relationship attribute filter(s)
+sortBy: attribute to which results are sorted by
+sortOrder: sorting order of results
+marker: add either offset or marker, value of marker for first
page will be '*'
+ and value of nextMarker: in the response will be input
of marker for subsequent pages`}
+</SyntaxHighlighter>
+
+Attribute based filtering can be done on multiple attributes with AND/OR
conditions.
+
+**Real time example of Relationship Search**
+
+Consider use-case of social media based application '*Instagram*' and its
basic functionalities like:
+
+* *User* creates *Posts*
+* *User* reacts to *Posts*
+* *User* creates *Highlight*
+* *Posts* added in *Highlight*
+
+Below is the Metadata Model highlighting above use-case
+
+<Img src={`/images/twiki/relationship_search_model.png`} height="500"
width="840"/>
+
+Example:
+Consider a user Ajay, who had uploaded his celebratory Diwali photo as a post.
Users like Divya, Rahul and many others reacted with a like to his post.
+
+Now the user Ajay wants to find out the number of 'like' reactions to his post.
+
+For his search, he will focus on 'user_post' relationship type and filter
those with attributes where reactions have 'like' in them and in which the post
name has 'Diwali' in it.
+
+**Example of filtering (for user_post attributes)**
+
+<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
+{` {
+ "relationshipName": "user_post",
+ "excludeDeletedEntities": true,
+ "offset": 0,
+ "limit": 25,
+ "relationshipFilters": {
+ "condition": "AND",
+ "criterion": [
+ {
+ "attributeName": "reaction",
+ "operator": "eq",
+ "attributeValue": "like"
+ },
+ {
+ "attributeName": "post_name",
+ "operator": "contains",
+ "attributeValue": "Diwali"
+ }
+ ]
+ }
+ }`}
+</SyntaxHighlighter>
+
+**Supported operators for filtering**
+
+Same operators as basic search
+
+**CURL Samples**
+
+<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
+{`curl -sivk -g
+ -u <user>:<password>
+ -X POST
+ -d '{
+ "relationshipName": "user_post",
+ "excludeDeletedEntities": true,
+ "offset": 0,
+ "limit": 25,
+ "relationshipFilters": {
+ "condition": "AND",
+ "criterion": [
+ {
+ "attributeName": "reaction",
+ "operator": "eq",
+ "attributeValue": "like"
+ },
+ {
+ "attributeName": "post_name",
+ "operator": "contains",
+ "attributeValue": "Diwali"
+ }
+ ]
+ }
+ }'
+ <protocol>://<atlas_host>:<atlas_port>/api/atlas/v2/search/relations`}
+</SyntaxHighlighter>
diff --git a/docs/src/resources/images/twiki/relationship_search_model.png
b/docs/src/resources/images/twiki/relationship_search_model.png
new file mode 100644
index 000000000..5d155d4cc
Binary files /dev/null and
b/docs/src/resources/images/twiki/relationship_search_model.png differ
diff --git
a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java
b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java
index 2edca64d7..2273e9f35 100644
---
a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java
+++
b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java
@@ -59,6 +59,36 @@ public interface AtlasIndexQuery<V, E> {
*/
Long vertexTotals();
+ /**
+ * Gets the query results.
+ *
+ * @return
+ */
+ Iterator<Result<V, E>> edges();
+
+ /**
+ * Gets the sorted query results
+ * @param offset starting offset
+ * @param limit max number of results
+ * @param sortBy sort attribute
+ * @param sortOrder sorting order asc, desc
+ * @return
+ */
+ Iterator<Result<V, E>> edges(int offset, int limit, String sortBy, Order
sortOrder);
+
+ /**
+ * Gets the query results
+ * @param offset starting offset
+ * @param limit max number of results
+ * @return
+ */
+ Iterator<Result<V, E>> edges(int offset, int limit);
+
+ /**
+ * Gets the total count of query results
+ * @return
+ */
+ Long edgeTotals();
/**
* Query result from an index query.
*
@@ -72,6 +102,11 @@ public interface AtlasIndexQuery<V, E> {
*/
AtlasVertex<V, E> getVertex();
+ /**
+ * Gets the edge for this result.
+ */
+ AtlasEdge<V, E> getEdge();
+
/**
* Gets the score for this result.
*
diff --git
a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQuery.java
b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQuery.java
index ff3fd4a78..c5b642a71 100644
---
a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQuery.java
+++
b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQuery.java
@@ -20,12 +20,14 @@ package org.apache.atlas.repository.graphdb.janus;
import java.util.Iterator;
import com.google.common.base.Preconditions;
+import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import com.google.common.base.Function;
import com.google.common.collect.Iterators;
import org.apache.tinkerpop.gremlin.process.traversal.Order;
+import org.janusgraph.core.JanusGraphEdge;
import org.janusgraph.core.JanusGraphIndexQuery;
import org.janusgraph.core.JanusGraphVertex;
@@ -106,24 +108,99 @@ public class AtlasJanusIndexQuery implements
AtlasIndexQuery<AtlasJanusVertex, A
return query.vertexTotals();
}
+ @Override
+ public Long edgeTotals() {
+ return query.edgeTotals();
+ }
+
+ @Override
+ public Iterator<Result<AtlasJanusVertex, AtlasJanusEdge>> edges() {
+ Iterator<JanusGraphIndexQuery.Result<JanusGraphEdge>> results =
query.edges().iterator();
+
+ Function<JanusGraphIndexQuery.Result<JanusGraphEdge>,
Result<AtlasJanusVertex, AtlasJanusEdge>> function =
+ new Function<JanusGraphIndexQuery.Result<JanusGraphEdge>,
Result<AtlasJanusVertex, AtlasJanusEdge>>() {
+
+ @Override
+ public Result<AtlasJanusVertex, AtlasJanusEdge>
apply(JanusGraphIndexQuery.Result<JanusGraphEdge> source) {
+ return new ResultImpl(source, null);
+ }
+ };
+
+ return Iterators.transform(results, function);
+ }
+
+ @Override
+ public Iterator<Result<AtlasJanusVertex, AtlasJanusEdge>> edges(int
offset, int limit) {
+ Preconditions.checkArgument(offset >=0, "Index offset should be
greater than or equals to 0");
+ Preconditions.checkArgument(limit >=0, "Index limit should be greater
than or equals to 0");
+ Iterator<JanusGraphIndexQuery.Result<JanusGraphEdge>> results = query
+ .offset(offset)
+ .limit(limit)
+ .edges().iterator();
+
+ Function<JanusGraphIndexQuery.Result<JanusGraphEdge>,
Result<AtlasJanusVertex, AtlasJanusEdge>> function =
+ new Function<JanusGraphIndexQuery.Result<JanusGraphEdge>,
Result<AtlasJanusVertex, AtlasJanusEdge>>() {
+
+ @Override
+ public Result<AtlasJanusVertex, AtlasJanusEdge>
apply(JanusGraphIndexQuery.Result<JanusGraphEdge> source) {
+ return new ResultImpl(source, null);
+ }
+ };
+
+ return Iterators.transform(results, function);
+ }
+
+ @Override
+ public Iterator<Result<AtlasJanusVertex, AtlasJanusEdge>> edges(int
offset, int limit, String sortBy, Order sortOrder) {
+ Preconditions.checkArgument(offset >=0, "Index offset should be
greater than or equals to 0");
+ Preconditions.checkArgument(limit >=0, "Index limit should be greater
than or equals to 0");
+
+ Iterator<JanusGraphIndexQuery.Result<JanusGraphEdge>> results = query
+ .orderBy(sortBy, sortOrder)
+ .offset(offset)
+ .limit(limit)
+ .edges().iterator();
+
+ Function<JanusGraphIndexQuery.Result<JanusGraphEdge>,
Result<AtlasJanusVertex, AtlasJanusEdge>> function =
+ new Function<JanusGraphIndexQuery.Result<JanusGraphEdge>,
Result<AtlasJanusVertex, AtlasJanusEdge>>() {
+
+ @Override
+ public Result<AtlasJanusVertex, AtlasJanusEdge>
apply(JanusGraphIndexQuery.Result<JanusGraphEdge> source) {
+ return new ResultImpl(source, null);
+ }
+ };
+ return Iterators.transform(results, function);
+ }
+
+
/**
* Janus implementation of AtlasIndexQuery.Result.
*/
public final class ResultImpl implements
AtlasIndexQuery.Result<AtlasJanusVertex, AtlasJanusEdge> {
- private JanusGraphIndexQuery.Result<JanusGraphVertex> source;
+ private JanusGraphIndexQuery.Result<JanusGraphVertex> vertex;
+ private JanusGraphIndexQuery.Result<JanusGraphEdge> edge;
public ResultImpl(JanusGraphIndexQuery.Result<JanusGraphVertex>
source) {
- this.source = source;
+ this.vertex = source;
+ }
+
+ public ResultImpl(JanusGraphIndexQuery.Result<JanusGraphEdge> source,
AtlasJanusEdge edge) {
+ this.edge = source;
}
@Override
public AtlasVertex<AtlasJanusVertex, AtlasJanusEdge> getVertex() {
- return GraphDbObjectFactory.createVertex(graph,
source.getElement());
+ return GraphDbObjectFactory.createVertex(graph,
vertex.getElement());
+ }
+
+ @Override
+ public AtlasEdge<AtlasJanusVertex, AtlasJanusEdge> getEdge() {
+ return GraphDbObjectFactory.createEdge(graph, edge.getElement());
}
@Override
public double getScore() {
- return source.getScore();
+ return vertex.getScore();
}
}
}
diff --git
a/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java
b/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java
index ce0f84b6e..c6027b101 100644
--- a/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java
+++ b/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java
@@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import org.apache.atlas.model.instance.AtlasEntityHeader;
+import org.apache.atlas.model.instance.AtlasRelationshipHeader;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
@@ -49,6 +50,7 @@ public class AtlasSearchResult implements Serializable {
private String type;
private String classification;
private List<AtlasEntityHeader> entities;
+ private List<AtlasRelationshipHeader> relations;
private AttributeSearchResult attributes;
private List<AtlasFullTextResult> fullTextResult;
private Map<String, AtlasEntityHeader> referredEntities;
@@ -66,6 +68,7 @@ public class AtlasSearchResult implements Serializable {
setQueryType(queryType);
setSearchParameters(null);
setEntities(null);
+ setRelations(null);
setAttributes(null);
setFullTextResult(null);
setReferredEntities(null);
@@ -78,6 +81,7 @@ public class AtlasSearchResult implements Serializable {
setQueryText(searchParameters.getQuery());
setSearchParameters(searchParameters);
setEntities(null);
+ setRelations(null);
setAttributes(null);
setFullTextResult(null);
setReferredEntities(null);
@@ -112,6 +116,14 @@ public class AtlasSearchResult implements Serializable {
public void setEntities(List<AtlasEntityHeader> entities) { this.entities
= entities; }
+ public List<AtlasRelationshipHeader> getRelations() {
+ return relations;
+ }
+
+ public void setRelations(List<AtlasRelationshipHeader> relations) {
+ this.relations = relations;
+ }
+
public AttributeSearchResult getAttributes() { return attributes; }
public void setAttributes(AttributeSearchResult attributes) {
this.attributes = attributes; }
@@ -137,7 +149,7 @@ public class AtlasSearchResult implements Serializable {
public void setNextMarker(String nextMarker) { this.nextMarker =
nextMarker; }
@Override
- public int hashCode() { return Objects.hash(queryType, searchParameters,
queryText, type, classification, entities, attributes, fullTextResult,
referredEntities, nextMarker); }
+ public int hashCode() { return Objects.hash(queryType, searchParameters,
queryText, type, classification, entities, relations, attributes,
fullTextResult, referredEntities, nextMarker); }
@Override
public boolean equals(Object o) {
@@ -150,6 +162,7 @@ public class AtlasSearchResult implements Serializable {
Objects.equals(type, that.type) &&
Objects.equals(classification, that.classification) &&
Objects.equals(entities, that.entities) &&
+ Objects.equals(relations, that.relations) &&
Objects.equals(attributes, that.attributes) &&
Objects.equals(fullTextResult, that.fullTextResult) &&
Objects.equals(referredEntities, that.referredEntities) &&
@@ -183,6 +196,33 @@ public class AtlasSearchResult implements Serializable {
}
}
+ public void addRelation(AtlasRelationshipHeader relation) {
+ if (relations == null) {
+ relations = new ArrayList<>();
+ }
+
+ if (relations.isEmpty()) {
+ relations.add(relation);
+ } else {
+ removeRelation(relation);
+ relations.add(relation);
+ }
+ }
+
+ public void removeRelation(AtlasRelationshipHeader relation) {
+ List<AtlasRelationshipHeader> relations = this.relations;
+
+ if (CollectionUtils.isNotEmpty(relations)) {
+ Iterator<AtlasRelationshipHeader> iter = relations.iterator();
+ while (iter.hasNext()) {
+ AtlasRelationshipHeader currEntity = iter.next();
+ if (StringUtils.equals(currEntity.getGuid(),
relation.getGuid())) {
+ iter.remove();
+ }
+ }
+ }
+ }
+
@Override
public String toString() {
return "AtlasSearchResult{" +
@@ -192,6 +232,7 @@ public class AtlasSearchResult implements Serializable {
", type=" + type +
", classification=" + classification +
", entities=" + entities +
+ ", relations=" + relations +
", attributes=" + attributes +
", fullTextResult=" + fullTextResult +
", referredEntities=" + referredEntities +
diff --git
a/intg/src/main/java/org/apache/atlas/model/discovery/RelationshipSearchParameters.java
b/intg/src/main/java/org/apache/atlas/model/discovery/RelationshipSearchParameters.java
new file mode 100644
index 000000000..0f132514e
--- /dev/null
+++
b/intg/src/main/java/org/apache/atlas/model/discovery/RelationshipSearchParameters.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.model.discovery;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import org.apache.atlas.SortOrder;
+
+import java.io.Serializable;
+import java.util.Set;
+
+import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
+import static
com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
+
+@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility =
PUBLIC_ONLY, fieldVisibility = NONE)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class RelationshipSearchParameters implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private String relationshipName;
+ private SearchParameters.FilterCriteria relationshipFilters;
+ private boolean includeSubTypes;
+ private int offset;
+ private int limit;
+ private String sortBy;
+ private SortOrder sortOrder;
+ private String marker;
+
+
+ public String getRelationshipName() {
+ return relationshipName;
+ }
+
+ public void setRelationshipName(String relationshipName) {
+ this.relationshipName = relationshipName;
+ }
+
+ public SearchParameters.FilterCriteria getRelationshipFilters() {
+ return relationshipFilters;
+ }
+
+ public void setRelationshipFilters(SearchParameters.FilterCriteria
relationshipFilters) {
+ this.relationshipFilters = relationshipFilters;
+ }
+
+ public boolean isIncludeSubTypes() {
+ return includeSubTypes;
+ }
+
+ public void setIncludeSubTypes(boolean includeSubTypes) {
+ this.includeSubTypes = includeSubTypes;
+ }
+
+ public int getOffset() {
+ return offset;
+ }
+
+ public void setOffset(int offset) {
+ this.offset = offset;
+ }
+
+ public int getLimit() {
+ return limit;
+ }
+
+ public void setLimit(int limit) {
+ this.limit = limit;
+ }
+
+ public String getSortBy() {
+ return sortBy;
+ }
+
+ public void setSortBy(String sortBy) {
+ this.sortBy = sortBy;
+ }
+
+ public SortOrder getSortOrder() {
+ return sortOrder;
+ }
+
+ public void setSortOrder(SortOrder sortOrder) {
+ this.sortOrder = sortOrder;
+ }
+
+ public String getMarker() {
+ return marker;
+ }
+
+ public void setMarker(String marker) {
+ this.marker = marker;
+ }
+}
diff --git
a/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
b/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
index 8e68d0e82..1de1f122b 100644
--- a/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
+++ b/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
@@ -44,6 +44,7 @@ public class SearchParameters implements Serializable {
private String query;
private String typeName;
private String classification;
+ private String relationshipName;
private String termName;
private String sortBy;
private boolean excludeDeletedEntities;
@@ -58,6 +59,7 @@ public class SearchParameters implements Serializable {
private FilterCriteria entityFilters;
private FilterCriteria tagFilters;
+ private FilterCriteria relationshipFilters;
private Set<String> attributes;
private SortOrder sortOrder;
@@ -129,6 +131,14 @@ public class SearchParameters implements Serializable {
this.classification = classification;
}
+ public String getRelationshipName() {
+ return relationshipName;
+ }
+
+ public void setRelationshipName(String relationshipName) {
+ this.relationshipName = relationshipName;
+ }
+
/**
* @return True iff deleted entities are excluded
*/
@@ -260,6 +270,19 @@ public class SearchParameters implements Serializable {
this.tagFilters = tagFilters;
}
+ /**
+ * Relationship attribute filters for the relationship
+ * @return
+ */
+ public FilterCriteria getRelationshipFilters() { return
relationshipFilters; }
+
+ /**
+ * Filter the relationship on this criteria
+ * @param relationshipFilters
+ */
+ public void setRelationshipFilters(FilterCriteria relationshipFilters) {
this.relationshipFilters = relationshipFilters; }
+
+
public boolean getExcludeHeaderAttributes() {
return excludeHeaderAttributes;
}
diff --git
a/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationshipHeader.java
b/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationshipHeader.java
index ad3b98ed4..524a29d85 100644
---
a/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationshipHeader.java
+++
b/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationshipHeader.java
@@ -83,6 +83,14 @@ public class AtlasRelationshipHeader extends AtlasStruct
implements Serializable
}
}
+ public AtlasRelationshipHeader(AtlasRelationship relationship, boolean
withAttributes) {
+ this(relationship);
+
+ if (withAttributes) {
+ setAttributes(relationship.getAttributes());
+ }
+ }
+
public String getGuid() {
return guid;
diff --git
a/intg/src/main/java/org/apache/atlas/model/profile/AtlasUserSavedSearch.java
b/intg/src/main/java/org/apache/atlas/model/profile/AtlasUserSavedSearch.java
index 36ab3b11a..4f55f8ef2 100644
---
a/intg/src/main/java/org/apache/atlas/model/profile/AtlasUserSavedSearch.java
+++
b/intg/src/main/java/org/apache/atlas/model/profile/AtlasUserSavedSearch.java
@@ -38,10 +38,18 @@ public class AtlasUserSavedSearch extends
AtlasBaseModelObject implements Serial
public enum SavedSearchType {
BASIC,
- ADVANCED;
+ ADVANCED,
+ BASIC_RELATIONSHIP;
public static SavedSearchType to(String val) {
- return SavedSearchType.ADVANCED.name().equalsIgnoreCase(val) ?
SavedSearchType.ADVANCED : SavedSearchType.BASIC;
+ SavedSearchType searchType = SavedSearchType.BASIC;
+ if (SavedSearchType.ADVANCED.name().equalsIgnoreCase(val)) {
+ searchType = SavedSearchType.ADVANCED;
+ } else if
(SavedSearchType.BASIC_RELATIONSHIP.name().equalsIgnoreCase(val)) {
+ searchType = SavedSearchType.BASIC_RELATIONSHIP;
+ }
+
+ return searchType;
}
}
diff --git
a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
index 0f564c200..d94110004 100644
---
a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
+++
b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
@@ -79,6 +79,14 @@ public interface AtlasDiscoveryService {
*/
AtlasSearchResult searchWithParameters(SearchParameters searchParameters)
throws AtlasBaseException;
+ /**
+ * Search for relations (edges) matching the search criteria
+ * @param searchParameters Search criteria
+ * @return Matching Edge Entities
+ * @throws AtlasBaseException
+ */
+ AtlasSearchResult
searchRelationsWithParameters(RelationshipSearchParameters searchParameters)
throws AtlasBaseException;
+
/**
*
* @param guid unique ID of the entity.
diff --git
a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
index 4b113dbef..582d97542 100644
---
a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
+++
b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
@@ -34,10 +34,12 @@ import
org.apache.atlas.model.discovery.AtlasSearchResult.AtlasFullTextResult;
import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasQueryType;
import org.apache.atlas.model.discovery.AtlasSuggestionsResult;
import org.apache.atlas.model.discovery.QuickSearchParameters;
+import org.apache.atlas.model.discovery.RelationshipSearchParameters;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
+import org.apache.atlas.model.instance.AtlasRelationshipHeader;
import org.apache.atlas.model.profile.AtlasUserSavedSearch;
import org.apache.atlas.query.QueryParams;
import org.apache.atlas.query.executors.DSLQueryExecutor;
@@ -449,6 +451,46 @@ public class EntityDiscoveryService implements
AtlasDiscoveryService {
return searchWithSearchContext(new SearchContext(searchParameters,
typeRegistry, graph, indexer.getVertexIndexKeys()));
}
+ @Override
+ @GraphTransaction
+ public AtlasSearchResult
searchRelationsWithParameters(RelationshipSearchParameters searchParameters)
throws AtlasBaseException {
+ SearchContext searchContext = new
SearchContext(createSearchParameters(searchParameters),
+ typeRegistry,
+ graph,
+ null);
+ return searchRelationsWithSearchContext(searchContext);
+ }
+
+ private AtlasSearchResult searchRelationsWithSearchContext(SearchContext
searchContext) throws AtlasBaseException {
+ SearchParameters searchParameters =
searchContext.getSearchParameters();
+ AtlasSearchResult ret = new
AtlasSearchResult(searchParameters);
+ final QueryParams params =
QueryParams.getNormalizedParams(searchParameters.getLimit(),searchParameters.getOffset());
+
+ searchParameters.setLimit(params.limit());
+ searchParameters.setOffset(params.offset());
+
+ if (!searchContext.needRelationshipProcessor()) {
+ return ret;
+ }
+
+ RelationshipSearchProcessor rsp = new
RelationshipSearchProcessor(searchContext, indexer.getEdgeIndexKeys());
+ List<AtlasEdge> edges = rsp.executeEdges();
+
+ ret.setApproximateCount(rsp.getResultCount());
+
+ String nextMarker = rsp.getNextMarker();
+ if (StringUtils.isNotEmpty(nextMarker)) {
+ ret.setNextMarker(nextMarker);
+ }
+
+ for (AtlasEdge edge : edges) {
+ AtlasRelationshipHeader relation =
entityRetriever.mapEdgeToAtlasRelationshipHeader(edge);
+ ret.addRelation(relation);
+ }
+
+ return ret;
+ }
+
private AtlasSearchResult searchWithSearchContext(SearchContext
searchContext) throws AtlasBaseException {
SearchParameters searchParameters =
searchContext.getSearchParameters();
AtlasSearchResult ret = new
AtlasSearchResult(searchParameters);
@@ -953,6 +995,21 @@ public class EntityDiscoveryService implements
AtlasDiscoveryService {
return searchParameters;
}
+ public static SearchParameters
createSearchParameters(RelationshipSearchParameters
relationshipSearchParameters) {
+ SearchParameters searchParameters = new SearchParameters();
+
+
searchParameters.setRelationshipName(relationshipSearchParameters.getRelationshipName());
+
searchParameters.setIncludeSubTypes(relationshipSearchParameters.isIncludeSubTypes());
+ searchParameters.setLimit(relationshipSearchParameters.getLimit());
+ searchParameters.setOffset(relationshipSearchParameters.getOffset());
+
searchParameters.setRelationshipFilters(relationshipSearchParameters.getRelationshipFilters());
+ searchParameters.setSortBy(relationshipSearchParameters.getSortBy());
+
searchParameters.setSortOrder(relationshipSearchParameters.getSortOrder());
+ searchParameters.setMarker(relationshipSearchParameters.getMarker());
+
+ return searchParameters;
+ }
+
private String escapeTypeName(String typeName) {
String ret;
diff --git
a/repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java
b/repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java
index 1cd8786be..e8f51a3ab 100644
---
a/repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java
+++
b/repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java
@@ -96,4 +96,15 @@ public class GraphIndexQueryBuilder {
.append("\":").append(typeAndAllSubTypesQryStr).append(")");
}
}
+
+ void addTypeAndSubTypesQueryFilterEdges(StringBuilder indexQuery, String
typeAndAllSubTypesQryStr) {
+ if (indexQuery != null &&
StringUtils.isNotEmpty(typeAndAllSubTypesQryStr)) {
+ if (indexQuery.length() > 0) {
+ indexQuery.append(" AND ");
+ }
+
+ indexQuery.append("(").append(INDEX_SEARCH_PREFIX +
"\"").append(Constants.RELATIONSHIP_TYPE_PROPERTY_KEY)
+
.append("\":").append(typeAndAllSubTypesQryStr).append(")");
+ }
+ }
}
\ No newline at end of file
diff --git
a/repository/src/main/java/org/apache/atlas/discovery/RelationshipSearchProcessor.java
b/repository/src/main/java/org/apache/atlas/discovery/RelationshipSearchProcessor.java
new file mode 100644
index 000000000..e4c07988d
--- /dev/null
+++
b/repository/src/main/java/org/apache/atlas/discovery/RelationshipSearchProcessor.java
@@ -0,0 +1,210 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.discovery;
+
+import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.AtlasEdge;
+import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
+import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.type.AtlasRelationshipType;
+import org.apache.atlas.type.AtlasStructType;
+import org.apache.atlas.util.SearchPredicateUtil;
+import org.apache.atlas.utils.AtlasPerfTracer;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.Predicate;
+import org.apache.commons.collections.PredicateUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static
org.apache.atlas.repository.Constants.RELATIONSHIP_TYPE_PROPERTY_KEY;
+
+public class RelationshipSearchProcessor extends SearchProcessor {
+ private static final Logger LOG =
LoggerFactory.getLogger(RelationshipSearchProcessor.class);
+ private static final Logger PERF_LOG =
AtlasPerfTracer.getPerfLogger("RelationshipSearchProcessor");
+
+ private final AtlasIndexQuery indexQuery;
+ private final AtlasGraphQuery graphQuery;
+
+ RelationshipSearchProcessor(SearchContext context, Set<String>
indexedKeys) {
+ super(context);
+ context.setEdgeIndexKeys(indexedKeys);
+
+ final Set<AtlasRelationshipType> types =
context.getRelationshipTypes();
+ final SearchParameters.FilterCriteria filterCriteria =
context.getSearchParameters().getRelationshipFilters();
+ final Set<String> indexAttributes = new HashSet<>();
+ final Set<String> graphAttributes = new HashSet<>();
+ final Set<String> allAttributes = new HashSet<>();
+ final Set<String> typeNames =
CollectionUtils.isNotEmpty(types) ?
types.stream().map(AtlasRelationshipType::getTypeName).collect(Collectors.toSet())
: null;
+ final String typeAndSubTypesQryStr =
AtlasStructType.AtlasAttribute.escapeIndexQueryValue(typeNames, true);
+ final Predicate typeNamePredicate =
SearchPredicateUtil.generateIsRelationshipEdgePredicate(context.getTypeRegistry());
+ inMemoryPredicate = typeNamePredicate;
+
+ processSearchAttributes(types, filterCriteria, indexAttributes,
graphAttributes, allAttributes);
+
+ final boolean typeSearchByIndex = typeAndSubTypesQryStr.length() <=
MAX_QUERY_STR_LENGTH_TYPES;
+ final boolean attrSearchByIndex =
CollectionUtils.isNotEmpty(indexAttributes) && canApplyIndexFilter(types,
filterCriteria, false);
+
+ //index query
+ StringBuilder indexQuery = new StringBuilder();
+
+ if (typeSearchByIndex) {
+
graphIndexQueryBuilder.addTypeAndSubTypesQueryFilterEdges(indexQuery,
typeAndSubTypesQryStr);
+ }
+
+ if (attrSearchByIndex) {
+ constructFilterQuery(indexQuery, types, filterCriteria,
indexAttributes);
+
+ Predicate attributePredicate = constructInMemoryPredicate(types,
filterCriteria, indexAttributes);
+ if (attributePredicate != null) {
+ inMemoryPredicate =
PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate);
+ }
+ } else {
+ graphAttributes.addAll(indexAttributes);
+ }
+
+ if (indexQuery.length() > 0) {
+
+ String indexQueryString =
STRAY_AND_PATTERN.matcher(indexQuery).replaceAll(")");
+ indexQueryString =
STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")");
+ indexQueryString =
STRAY_ELIPSIS_PATTERN.matcher(indexQueryString).replaceAll("");
+
+ this.indexQuery =
context.getGraph().indexQuery(Constants.EDGE_INDEX, indexQueryString);
+ } else {
+ this.indexQuery = null;
+ }
+
+ //graph query
+ if (CollectionUtils.isNotEmpty(graphAttributes) || !typeSearchByIndex)
{
+ AtlasGraphQuery query = context.getGraph().query();
+
+ if (!typeSearchByIndex) {
+ query.in(RELATIONSHIP_TYPE_PROPERTY_KEY, types);
+ }
+ graphQuery = toGraphFilterQuery(types, filterCriteria,
graphAttributes, query);
+
+ Predicate attributePredicate = constructInMemoryPredicate(types,
filterCriteria, graphAttributes);
+ if (attributePredicate != null) {
+ inMemoryPredicate =
PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate);
+ }
+ } else {
+ graphQuery = null;
+ }
+
+ }
+
+ @Override
+ public List<AtlasVertex> execute() {
+ return null;
+ }
+
+ public List<AtlasEdge> executeEdges() {
+ List<AtlasEdge> ret = new ArrayList<>();
+
+ AtlasPerfTracer perf = null;
+ if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
+ perf = AtlasPerfTracer.getPerfTracer(PERF_LOG,
"RelationshipSearchProcessor.execute(" + context + ")");
+ }
+
+ try {
+ final int limit = context.getSearchParameters().getLimit();
+ final Integer marker = context.getMarker();
+ final int startIdx = marker != null ? marker :
context.getSearchParameters().getOffset();
+
+ // when subsequent filtering stages are involved, query should
start at 0 even though startIdx can be higher
+ // first 'startIdx' number of entries will be ignored
+ // if marker is provided, start query with marker offset
+ int qryOffset;
+ if (marker != null) {
+ qryOffset = marker;
+ } else {
+ qryOffset = (graphQuery != null && indexQuery != null) ? 0 :
startIdx;
+ }
+ int resultIdx = qryOffset;
+
+ LinkedHashMap<Integer, AtlasEdge> offsetEdgeMap = new
LinkedHashMap<>();
+
+ for (; ret.size() < limit; qryOffset += limit) {
+ offsetEdgeMap.clear();
+
+ if (context.terminateSearch()) {
+ LOG.warn("query terminated: {}",
context.getSearchParameters());
+
+ break;
+ }
+
+ final boolean isLastResultPage;
+
+ if (indexQuery != null) {
+ Iterator<AtlasIndexQuery.Result> idxQueryResult =
executeIndexQueryForEdge(context, indexQuery, qryOffset, limit);
+ offsetEdgeMap =
getEdgesFromIndexQueryResult(idxQueryResult, offsetEdgeMap, qryOffset);
+
+ } else {
+ Iterator<AtlasEdge> queryResult =
graphQuery.edges(qryOffset, limit).iterator();
+ offsetEdgeMap = getEdges(queryResult, offsetEdgeMap,
qryOffset);
+ }
+
+ isLastResultPage = offsetEdgeMap.size() < limit;
+
+ // Do in-memory filtering
+ offsetEdgeMap = offsetEdgeMap.entrySet()
+ .stream()
+ .filter(x -> inMemoryPredicate.evaluate(x.getValue()))
+ .collect(Collectors.toMap(Map.Entry::getKey,
Map.Entry::getValue, (x, y) -> y, LinkedHashMap::new));
+
+ resultIdx = collectResultEdges(ret, startIdx, limit,
resultIdx, offsetEdgeMap, marker);
+
+ if (isLastResultPage) {
+ resultIdx = SearchContext.MarkerUtil.MARKER_END - 1;
+ break;
+ }
+ }
+
+ if (marker != null) {
+ nextOffset = resultIdx + 1;
+ }
+
+ } finally {
+ AtlasPerfTracer.log(perf);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== RelationshipSearchProcessor.execute({}):
ret.size()={}", context, ret.size());
+ }
+ return ret;
+ }
+
+ @Override
+ public long getResultCount() {
+ if (indexQuery != null) {
+ return indexQuery.edgeTotals();
+ } else {
+ return -1L;
+ }
+ }
+}
diff --git
a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
index b8976e079..9a9a5512a 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
@@ -36,6 +36,7 @@ import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasBuiltInTypes;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasRelationshipType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasType;
@@ -78,9 +79,12 @@ public class SearchContext {
private final AtlasGraph graph;
private final Set<AtlasEntityType> entityTypes;
private final Set<String> indexedKeys;
+ private Set<String> edgeIndexKeys;
private final Set<String> entityAttributes;
+ private final Set<String> relationAttributes;
private final SearchParameters searchParameters;
private final Set<AtlasClassificationType> classificationTypes;
+ private final Set<AtlasRelationshipType> relationshipTypes;
private final Set<String> classificationNames;
private final Set<String> typeAndSubTypes;
private final Set<String> classificationTypeAndSubTypes;
@@ -105,9 +109,11 @@ public class SearchContext {
this.graph = graph;
this.indexedKeys = indexedKeys;
this.entityAttributes = new HashSet<>();
+ this.relationAttributes = new HashSet<>();
this.entityTypes =
getEntityTypes(searchParameters.getTypeName());
this.classificationNames =
getClassificationNames(searchParameters.getClassification());
this.classificationTypes =
getClassificationTypes(this.classificationNames);
+ this.relationshipTypes =
getRelationshipTypes(searchParameters.getRelationshipName());
AtlasVertex glossaryTermVertex =
getGlossaryTermVertex(searchParameters.getTermName());
@@ -143,6 +149,13 @@ public class SearchContext {
}
}
+ // Invalid relationship attributes will raise an exception with 400
error code
+ if (CollectionUtils.isNotEmpty(relationshipTypes)) {
+ for (AtlasRelationshipType relationshipType : relationshipTypes) {
+ validateAttributes(relationshipType,
searchParameters.getRelationshipFilters());
+ }
+ }
+
if (StringUtils.isNotEmpty(searchParameters.getMarker())) {
marker = MarkerUtil.decodeMarker(searchParameters);
}
@@ -239,8 +252,16 @@ public class SearchContext {
public Set<String> getIndexedKeys() { return indexedKeys; }
+ public void setEdgeIndexKeys(Set<String> edgeIndexKeys) {
+ this.edgeIndexKeys = edgeIndexKeys;
+ }
+
+ public Set<String> getEdgeIndexKeys() { return edgeIndexKeys; }
+
public Set<String> getEntityAttributes() { return entityAttributes; }
+ public Set<String> getRelationAttributes() { return relationAttributes; }
+
public Set<AtlasClassificationType> getClassificationTypes() { return
classificationTypes; }
public Set<String> getEntityTypeNames() { return typeAndSubTypes; }
@@ -259,6 +280,8 @@ public class SearchContext {
public Integer getMarker() { return marker; }
+ public Set<AtlasRelationshipType> getRelationshipTypes() { return
relationshipTypes; }
+
public boolean includeEntityType(String entityType) {
return typeAndSubTypes.isEmpty() ||
typeAndSubTypes.contains(entityType);
}
@@ -304,6 +327,10 @@ public class SearchContext {
return StringUtils.isNotEmpty(searchParameters.getQuery());
}
+ boolean needRelationshipProcessor() {
+ return CollectionUtils.isNotEmpty(relationshipTypes);
+ }
+
boolean needClassificationProcessor() {
return (CollectionUtils.isNotEmpty(classificationTypes) &&
(CollectionUtils.isEmpty(entityTypes) ||
hasAttributeFilter(searchParameters.getTagFilters()))) || isWildCardSearch() ;
}
@@ -561,6 +588,35 @@ public class SearchContext {
return
typeRegistry.getEntityTypeByName(TermSearchProcessor.ATLAS_GLOSSARY_TERM_ENTITY_TYPE);
}
+ private Set<AtlasRelationshipType> getRelationshipTypes(String
relationship) throws AtlasBaseException {
+ Set<AtlasRelationshipType> relationshipTypes = null;
+ //split multiple typeNames by comma
+ if (StringUtils.isNotEmpty(relationship)) {
+
+ String[] types = relationship.split(TYPENAME_DELIMITER);
+ Set<String> typeNames = new HashSet<>(Arrays.asList(types));
+ relationshipTypes = typeNames.stream().map(n ->
+
typeRegistry.getRelationshipTypeByName(n)).filter(Objects::nonNull).collect(Collectors.toSet());
+
+ // Validate if the type name is incorrect
+ if (CollectionUtils.isEmpty(relationshipTypes)) {
+ throw new
AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME,relationship);
+
+ } else if (relationshipTypes.size() != typeNames.size()) {
+ Set<String> validEntityTypes = new HashSet<>();
+ for (AtlasRelationshipType entityType : relationshipTypes) {
+ validEntityTypes.add(entityType.getTypeName());
+ }
+
+ typeNames.removeAll(validEntityTypes);
+
+ LOG.info("Could not search for {} , invalid typeNames",
String.join(TYPENAME_DELIMITER, typeNames));
+ }
+ }
+
+ return relationshipTypes;
+ }
+
public static class MarkerUtil {
private final static int IDX_HASH_CODE = 0;
private final static int IDX_OFFSET = 1;
diff --git
a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index f69dc4215..7b36c6c75 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -29,6 +29,7 @@ import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphHelper;
+import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
@@ -37,7 +38,6 @@ import org.apache.atlas.type.*;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.util.AtlasGremlinQueryProvider;
import org.apache.atlas.util.SearchPredicateUtil;
-import org.apache.atlas.util.SearchPredicateUtil.*;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.collections.Predicate;
@@ -208,6 +208,26 @@ public abstract class SearchProcessor {
return marker == null ? resultIdx : lastOffset;
}
+ protected int collectResultEdges(final List<AtlasEdge> ret, final int
startIdx, final int limit, int resultIdx, final Map<Integer, AtlasEdge>
offsetEdgeMap, Integer marker) {
+ int lastOffset = resultIdx;
+
+ for (Map.Entry<Integer, AtlasEdge> offsetToEdge :
offsetEdgeMap.entrySet()) {
+ resultIdx++;
+
+ if (resultIdx <= startIdx) {
+ continue;
+ }
+
+ lastOffset = offsetToEdge.getKey();
+ ret.add(offsetToEdge.getValue());
+
+ if (ret.size() == limit) {
+ break;
+ }
+ }
+ return marker == null ? resultIdx : lastOffset;
+ }
+
public LinkedHashMap<Integer, AtlasVertex> filter(LinkedHashMap<Integer,
AtlasVertex> offsetEntityVertexMap) {
if (nextProcessor != null &&
MapUtils.isNotEmpty(offsetEntityVertexMap)) {
return nextProcessor.filter(offsetEntityVertexMap);
@@ -324,6 +344,10 @@ public abstract class SearchProcessor {
context.getEntityAttributes().add(attributeName);
}
+ if (structType instanceof AtlasRelationshipType) {
+ context.getRelationAttributes().add(attributeName);
+ }
+
allAttributes.add(qualifiedName);
}
} catch (AtlasBaseException e) {
@@ -490,7 +514,8 @@ public abstract class SearchProcessor {
String typeName = attributeType.getTypeName();
String qualifiedName =
structType.getVertexPropertyName(attributeName);
Set<String> indexedKeys = context.getIndexedKeys();
- boolean ret = indexedKeys != null &&
indexedKeys.contains(qualifiedName);
+ Set<String> edgeIndexedKeys = context.getEdgeIndexKeys();
+ boolean ret = (indexedKeys != null &&
(indexedKeys.contains(qualifiedName)) || (edgeIndexedKeys != null &&
edgeIndexedKeys.contains(qualifiedName)));
SearchParameters.Operator operator =
filterCriteria.getOperator();
AtlasStructDef.AtlasAttributeDef.IndexType indexType =
structType.getAttributeDef(attributeName).getIndexType();
@@ -1238,6 +1263,18 @@ public abstract class SearchProcessor {
return offsetEntityVertexMap;
}
+ protected LinkedHashMap<Integer, AtlasEdge>
getEdgesFromIndexQueryResult(Iterator<AtlasIndexQuery.Result> idxQueryResult,
LinkedHashMap<Integer, AtlasEdge> offsetEdgeMap, int qryOffset) {
+ if (idxQueryResult != null) {
+ while (idxQueryResult.hasNext()) {
+ AtlasEdge edge = idxQueryResult.next().getEdge();
+
+ offsetEdgeMap.put(qryOffset++, edge);
+ }
+ }
+
+ return offsetEdgeMap;
+ }
+
protected Collection<AtlasVertex> getVertices(Iterator<AtlasVertex>
iterator, Collection<AtlasVertex> vertices) {
if (iterator != null) {
while (iterator.hasNext()) {
@@ -1262,6 +1299,18 @@ public abstract class SearchProcessor {
return offsetEntityVertexMap;
}
+ protected LinkedHashMap<Integer, AtlasEdge> getEdges(Iterator<AtlasEdge>
iterator, LinkedHashMap<Integer, AtlasEdge> offsetEdgeMap, int qryOffset) {
+ if (iterator != null) {
+ while (iterator.hasNext()) {
+ AtlasEdge edge = iterator.next();
+
+ offsetEdgeMap.put(qryOffset++, edge);
+ }
+ }
+
+ return offsetEdgeMap;
+ }
+
protected Set<String> getGuids(List<AtlasVertex> vertices) {
Set<String> ret = new HashSet<>();
@@ -1315,4 +1364,13 @@ public abstract class SearchProcessor {
}
return indexQuery.vertices(qryOffset, limit);
}
+
+ protected static Iterator<AtlasIndexQuery.Result>
executeIndexQueryForEdge(SearchContext context, AtlasIndexQuery indexQuery, int
qryOffset, int limit) {
+ String sortBy = getSortByAttribute(context);
+ if (sortBy != null && !sortBy.isEmpty()) {
+ Order sortOrder = getSortOrderAttribute(context);
+ return indexQuery.edges(qryOffset, limit, sortBy, sortOrder);
+ }
+ return indexQuery.edges(qryOffset, limit);
+ }
}
diff --git
a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
index 9924b2e4c..5780f2ea0 100755
---
a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
+++
b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
@@ -94,8 +94,10 @@ public class GraphBackedSearchIndexer implements
SearchIndexer, ActiveStateChang
//allows injection of a dummy graph for testing
private IAtlasGraphProvider provider;
- private boolean recomputeIndexedKeys = true;
- private Set<String> vertexIndexKeys = new HashSet<>();
+ private boolean recomputeIndexedKeys = true;
+ private boolean recomputeEdgeIndexedKeys = true;
+ private Set<String> vertexIndexKeys = new HashSet<>();
+ private Set<String> edgeIndexKeys = new HashSet<>();
public static boolean isValidSearchWeight(int searchWeight) {
if (searchWeight != -1 ) {
@@ -280,6 +282,46 @@ public class GraphBackedSearchIndexer implements
SearchIndexer, ActiveStateChang
return vertexIndexKeys;
}
+ public Set<String> getEdgeIndexKeys() {
+ if (recomputeEdgeIndexedKeys) {
+ AtlasGraphManagement management = null;
+
+ try {
+ management = provider.get().getManagementSystem();
+
+ if (management != null) {
+ AtlasGraphIndex edgeIndex =
management.getGraphIndex(EDGE_INDEX);
+
+ if (edgeIndex != null) {
+ recomputeEdgeIndexedKeys = false;
+
+ Set<String> indexKeys = new HashSet<>();
+
+ for (AtlasPropertyKey fieldKey :
edgeIndex.getFieldKeys()) {
+ indexKeys.add(fieldKey.getName());
+ }
+
+ edgeIndexKeys = indexKeys;
+ }
+
+ management.commit();
+ }
+ } catch (Exception excp) {
+ LOG.error("getEdgeIndexKeys(): failed to get indexedKeys from
graph", excp);
+
+ if (management != null) {
+ try {
+ management.rollback();
+ } catch (Exception e) {
+ LOG.error("getEdgeIndexKeys(): rollback failed", e);
+ }
+ }
+ }
+ }
+
+ return edgeIndexKeys;
+ }
+
/**
* Initializes the indices for the graph - create indices for Global
AtlasVertex Keys
*/
@@ -371,6 +413,7 @@ public class GraphBackedSearchIndexer implements
SearchIndexer, ActiveStateChang
// create edge indexes
createEdgeIndex(management, RELATIONSHIP_GUID_PROPERTY_KEY,
String.class, SINGLE, true);
createEdgeIndex(management, EDGE_ID_IN_IMPORT_KEY, String.class,
SINGLE, true);
+ createEdgeIndex(management, RELATIONSHIP_TYPE_PROPERTY_KEY,
String.class, SINGLE, true);
// create fulltext indexes
createFullTextIndex(management, ENTITY_TEXT_PROPERTY_KEY,
String.class, SINGLE);
@@ -582,7 +625,7 @@ public class GraphBackedSearchIndexer implements
SearchIndexer, ActiveStateChang
} else if (isBuiltInType) {
if (isRelationshipType(atlasType)) {
- createEdgeIndex(management, propertyName,
getPrimitiveClass(attribTypeName), cardinality, false);
+ createEdgeIndex(management, propertyName,
getPrimitiveClass(attribTypeName), cardinality, isIndexable);
} else {
Class primitiveClassType =
getPrimitiveClass(attribTypeName);
boolean isStringField = false;
@@ -836,7 +879,7 @@ public class GraphBackedSearchIndexer implements
SearchIndexer, ActiveStateChang
}
- private void createEdgeIndex(AtlasGraphManagement management, String
propertyName, Class propertyClass,
+ public void createEdgeIndex(AtlasGraphManagement management, String
propertyName, Class propertyClass,
AtlasCardinality cardinality, boolean
createCompositeIndex) {
if (propertyName != null) {
AtlasPropertyKey propertyKey =
management.getPropertyKey(propertyName);
diff --git
a/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchManager.java
b/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchManager.java
index 44cd8efff..a9774ae58 100644
---
a/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchManager.java
+++
b/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchManager.java
@@ -96,6 +96,7 @@ public class AtlasPatchManager {
handlers.add(new ReIndexPatch(context));
handlers.add(new ProcessNamePatch(context));
handlers.add(new UpdateCompositeIndexStatusPatch(context));
+ handlers.add(new RelationshipTypeNamePatch(context));
LOG.info("<== AtlasPatchManager.init()");
}
diff --git
a/repository/src/main/java/org/apache/atlas/repository/patches/EdgePatchProcessor.java
b/repository/src/main/java/org/apache/atlas/repository/patches/EdgePatchProcessor.java
new file mode 100644
index 000000000..5721041a8
--- /dev/null
+++
b/repository/src/main/java/org/apache/atlas/repository/patches/EdgePatchProcessor.java
@@ -0,0 +1,213 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.patches;
+
+import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.pc.WorkItemBuilder;
+import org.apache.atlas.pc.WorkItemConsumer;
+import org.apache.atlas.pc.WorkItemManager;
+import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
+import org.apache.atlas.repository.graphdb.AtlasEdge;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.type.AtlasRelationshipType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.commons.configuration.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
+
+public abstract class EdgePatchProcessor {
+ private static final Logger LOG =
LoggerFactory.getLogger(EdgePatchProcessor.class);
+
+ private static final String NUM_WORKERS_PROPERTY =
"atlas.patch.numWorkers";
+ private static final String BATCH_SIZE_PROPERTY = "atlas.patch.batchSize";
+ private static final String ATLAS_SOLR_SHARDS = "ATLAS_SOLR_SHARDS";
+ private static final String WORKER_NAME_PREFIX = "patchWorkItem";
+ public static final int NUM_WORKERS;
+ public static final int BATCH_SIZE;
+
+ private final AtlasGraph graph;
+ private final GraphBackedSearchIndexer indexer;
+ private final AtlasTypeRegistry typeRegistry;
+
+ static {
+ int numWorkers = 3;
+ int batchSize = 300;
+
+ try {
+ Configuration config = ApplicationProperties.get();
+
+ numWorkers = config.getInt(NUM_WORKERS_PROPERTY,
config.getInt(ATLAS_SOLR_SHARDS, 1) * 3);
+ batchSize = config.getInt(BATCH_SIZE_PROPERTY, 300);
+
+ LOG.info("EdgePatchProcessor: {}={}, {}={}", NUM_WORKERS_PROPERTY,
numWorkers, BATCH_SIZE_PROPERTY, batchSize);
+ } catch (Exception e) {
+ LOG.error("Error retrieving configuration.", e);
+ }
+
+ NUM_WORKERS = numWorkers;
+ BATCH_SIZE = batchSize;
+ }
+
+ public EdgePatchProcessor(PatchContext context) {
+ this.graph = context.getGraph();
+ this.indexer = context.getIndexer();
+ this.typeRegistry = context.getTypeRegistry();
+ }
+
+ public AtlasGraph getGraph() {
+ return graph;
+ }
+
+ public GraphBackedSearchIndexer getIndexer() {
+ return indexer;
+ }
+
+ public AtlasTypeRegistry getTypeRegistry() {
+ return typeRegistry;
+ }
+
+ public void apply() throws AtlasBaseException {
+ prepareForExecution();
+ execute();
+ }
+
+ protected abstract void prepareForExecution() throws AtlasBaseException;
+
+ protected abstract void submitEdgesToUpdate(WorkItemManager manager);
+
+ protected abstract void processEdgesItem(String edgeId, AtlasEdge edge,
String typeName, AtlasRelationshipType type) throws AtlasBaseException;
+
+ private void execute() {
+ WorkItemManager manager = new WorkItemManager(new
ConsumerBuilder(graph, typeRegistry, this),
+ WORKER_NAME_PREFIX, BATCH_SIZE, NUM_WORKERS, false);
+
+ try {
+ submitEdgesToUpdate(manager);
+
+ manager.drain();
+ } finally {
+ try {
+ manager.shutdown();
+ } catch (InterruptedException e) {
+ LOG.error("EdgePatchProcessor.execute(): interrupted during
WorkItemManager shutdown.", e);
+ }
+ }
+ }
+
+ private static class ConsumerBuilder implements WorkItemBuilder<Consumer,
String> {
+ private final AtlasTypeRegistry typeRegistry;
+ private final AtlasGraph graph;
+ private final EdgePatchProcessor patchItemProcessor;
+
+ public ConsumerBuilder(AtlasGraph graph, AtlasTypeRegistry
typeRegistry, EdgePatchProcessor patchItemProcessor) {
+ this.graph = graph;
+ this.typeRegistry = typeRegistry;
+ this.patchItemProcessor = patchItemProcessor;
+ }
+
+ @Override
+ public Consumer build(BlockingQueue<String> queue) {
+ return new Consumer(graph, typeRegistry, queue,
patchItemProcessor);
+ }
+ }
+
+ private static class Consumer extends WorkItemConsumer<String> {
+ private int MAX_COMMIT_RETRY_COUNT = 3;
+ private final AtlasGraph graph;
+ private final AtlasTypeRegistry typeRegistry;
+
+ private final AtomicLong counter;
+ private final EdgePatchProcessor individualItemProcessor;
+
+ public Consumer(AtlasGraph graph, AtlasTypeRegistry typeRegistry,
BlockingQueue<String> queue, EdgePatchProcessor individualItemProcessor) {
+ super(queue);
+
+ this.graph = graph;
+ this.typeRegistry = typeRegistry;
+ this.counter = new AtomicLong(0);
+ this.individualItemProcessor = individualItemProcessor;
+ }
+
+ @Override
+ protected void doCommit() {
+ if (counter.get() % BATCH_SIZE == 0) {
+ LOG.info("Processed: {}", counter.get());
+
+ attemptCommit();
+ }
+ }
+
+ @Override
+ protected void commitDirty() {
+ attemptCommit();
+
+ LOG.info("Total: Commit: {}", counter.get());
+
+ super.commitDirty();
+ }
+
+ private void attemptCommit() {
+ for (int retryCount = 1; retryCount <= MAX_COMMIT_RETRY_COUNT;
retryCount++) {
+ try {
+ graph.commit();
+
+ break;
+ } catch (Exception ex) {
+ LOG.error("Commit exception: ", retryCount, ex);
+
+ try {
+ Thread.currentThread().sleep(300 * retryCount);
+ } catch (InterruptedException e) {
+ LOG.error("Commit exception: Pause: Interrputed!", e);
+ }
+ }
+ }
+ }
+
+ @Override
+ protected void processItem(String edgeId) {
+ counter.incrementAndGet();
+ AtlasEdge edge = graph.getEdge(edgeId);
+
+ if (edge == null) {
+ LOG.warn("processItem(edgeId={}): AtlasEdge not found!",
edgeId);
+
+ return;
+ }
+
+ String typeName =
edge.getProperty(ENTITY_TYPE_PROPERTY_KEY, String.class);
+ AtlasRelationshipType relationshipType =
typeRegistry.getRelationshipTypeByName(typeName);
+ if (relationshipType == null) {
+ return;
+ }
+
+ try {
+ individualItemProcessor.processEdgesItem(edgeId, edge,
typeName, relationshipType);
+ doCommit();
+ } catch (AtlasBaseException e) {
+ LOG.error("Error processing: {}", edgeId, e);
+ }
+ }
+ }
+}
diff --git
a/repository/src/main/java/org/apache/atlas/repository/patches/RelationshipTypeNamePatch.java
b/repository/src/main/java/org/apache/atlas/repository/patches/RelationshipTypeNamePatch.java
new file mode 100644
index 000000000..c3a9106fa
--- /dev/null
+++
b/repository/src/main/java/org/apache/atlas/repository/patches/RelationshipTypeNamePatch.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.patches;
+
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.pc.WorkItemManager;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.AtlasEdge;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
+import org.apache.atlas.type.AtlasRelationshipType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+
+import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.APPLIED;
+import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
+import static
org.apache.atlas.repository.Constants.RELATIONSHIP_TYPE_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY;
+
+
+public class RelationshipTypeNamePatch extends AtlasPatchHandler {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(RelationshipTypeNamePatch.class);
+
+ private static final String PATCH_ID = "JAVA_PATCH_0000_0011";
+ private static final String PATCH_DESCRIPTION = "Populates Relationship
typeName as like of Entity TypeName for all Edges.";
+
+ private final PatchContext context;
+
+ public RelationshipTypeNamePatch(PatchContext context) {
+ super(context.getPatchRegistry(), PATCH_ID, PATCH_DESCRIPTION);
+
+ this.context = context;
+ }
+
+ @Override
+ public void apply() throws AtlasBaseException {
+ EdgePatchProcessor patchProcessor = new
RelationshipTypeNamePatchProcessor(context);
+
+ patchProcessor.apply();
+
+ setStatus(APPLIED);
+
+ LOG.info("EdgePatchProcessor.apply(): patchId={}, status={}",
getPatchId(), getStatus());
+ }
+
+ public static class RelationshipTypeNamePatchProcessor extends
EdgePatchProcessor {
+
+ public RelationshipTypeNamePatchProcessor(PatchContext context) {
+ super(context);
+ }
+
+ @Override
+ protected void prepareForExecution() {
+ // relationship typeName mixed and composite index is already
created in GraphBackedSearchIndexer @Order(1)
+ }
+
+ @Override
+ protected void submitEdgesToUpdate(WorkItemManager manager) {
+
+ AtlasGraph graph = getGraph();
+
+ Iterable<AtlasEdge> iterable = graph.getEdges();
+ int count = 0;
+
+ for (Iterator<AtlasEdge> iter = iterable.iterator();
iter.hasNext(); ) {
+ AtlasEdge edge = iter.next();
+
+ if (edge.getProperty(ENTITY_TYPE_PROPERTY_KEY, String.class)
!= null) {
+ String edgeId = edge.getId().toString();
+
+ manager.checkProduce(edgeId);
+ count++;
+ }
+ }
+
+ LOG.info("found {} edges with typeName != null", count);
+ }
+
+ @Override
+ protected void processEdgesItem(String edgeId, AtlasEdge edge, String
typeName, AtlasRelationshipType type) {
+
+ edge.setProperty(RELATIONSHIP_TYPE_PROPERTY_KEY, typeName);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("processItem(typeName={}, edgeId={}): Done!",
typeName, edgeId);
+ }
+ }
+ }
+}
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
index 89e9422cd..f4d301568 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
@@ -775,6 +775,18 @@ public class AtlasTypeDefStoreInitializer implements
ActiveStateChangeHandler {
typeDefStore.updateStructDefByName(typeName, updatedDef);
+ ret = APPLIED;
+ } else if
(typeDef.getClass().equals(AtlasRelationshipDef.class)) {
+ AtlasRelationshipDef updatedDef = new
AtlasRelationshipDef((AtlasRelationshipDef)typeDef);
+
+ for (AtlasAttributeDef attributeDef :
patch.getAttributeDefs()) {
+ updatedDef.addAttribute(attributeDef);
+ }
+
+ updatedDef.setTypeVersion(patch.getUpdateToVersion());
+
+ typeDefStore.updateRelationshipDefByName(typeName,
updatedDef);
+
ret = APPLIED;
} else {
throw new
AtlasBaseException(AtlasErrorCode.PATCH_NOT_APPLICABLE_FOR_TYPE,
patch.getAction(), typeDef.getClass().getSimpleName());
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
index 2fce123f8..fe57f1485 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
@@ -70,6 +70,7 @@ import static
org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
import static
org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_DEFAULT;
import static
org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_PROPERTY;
import static
org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
+import static
org.apache.atlas.repository.Constants.RELATIONSHIP_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.SUPER_TYPES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TYPENAME_PROPERTY_KEY;
@@ -133,7 +134,11 @@ public class AtlasGraphUtilsV2 {
}
public static String getTypeName(AtlasElement element) {
- return element.getProperty(ENTITY_TYPE_PROPERTY_KEY, String.class);
+ if (element instanceof AtlasEdge) {
+ return element.getProperty(RELATIONSHIP_TYPE_PROPERTY_KEY,
String.class);
+ } else {
+ return element.getProperty(ENTITY_TYPE_PROPERTY_KEY, String.class);
+ }
}
public static String getEdgeLabel(String fromNode, String toNode) {
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
index a0fd71f9a..ef0313e02 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
@@ -74,6 +74,7 @@ import static
org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.
import static
org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.ONE_TO_TWO;
import static
org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.TWO_TO_ONE;
import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
+import static
org.apache.atlas.repository.Constants.RELATIONSHIP_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.HOME_ID_KEY;
import static org.apache.atlas.repository.Constants.PROVENANCE_TYPE_KEY;
import static
org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY;
@@ -386,6 +387,7 @@ public class AtlasRelationshipStoreV2 implements
AtlasRelationshipStore {
final String guid =
AtlasTypeUtil.isAssignedGuid(relationshipGuid) ? relationshipGuid :
UUID.randomUUID().toString();
AtlasGraphUtilsV2.setEncodedProperty(ret,
ENTITY_TYPE_PROPERTY_KEY, relationship.getTypeName());
+ AtlasGraphUtilsV2.setEncodedProperty(ret,
RELATIONSHIP_TYPE_PROPERTY_KEY, relationship.getTypeName());
AtlasGraphUtilsV2.setEncodedProperty(ret,
RELATIONSHIP_GUID_PROPERTY_KEY, guid);
AtlasGraphUtilsV2.setEncodedProperty(ret, HOME_ID_KEY,
relationship.getHomeId());
AtlasGraphUtilsV2.setEncodedProperty(ret,
VERSION_PROPERTY_KEY, getRelationshipVersion(relationship));
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
index a8fe5a762..0ee86735d 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
@@ -34,6 +34,7 @@ import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.AtlasRelatedObjectId;
import org.apache.atlas.model.instance.AtlasRelationship;
import
org.apache.atlas.model.instance.AtlasRelationship.AtlasRelationshipWithExtInfo;
+import org.apache.atlas.model.instance.AtlasRelationshipHeader;
import org.apache.atlas.model.instance.AtlasStruct;
import org.apache.atlas.model.typedef.AtlasRelationshipDef;
import org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags;
@@ -720,6 +721,16 @@ public class EntityGraphRetriever {
return mapVertexToAtlasEntityHeader(entityVertex,
Collections.<String>emptySet());
}
+ public AtlasRelationshipHeader mapEdgeToAtlasRelationshipHeader(AtlasEdge
edge) throws AtlasBaseException {
+
+ AtlasRelationshipWithExtInfo withExtInfo = new
AtlasRelationshipWithExtInfo();
+
+ mapSystemAttributes(edge, withExtInfo, false);
+ mapAttributes(edge, withExtInfo);
+
+ return new AtlasRelationshipHeader(withExtInfo.getRelationship(),
true);
+ }
+
private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex
entityVertex, Set<String> attributes) throws AtlasBaseException {
AtlasEntityHeader ret = new AtlasEntityHeader();
diff --git
a/repository/src/main/java/org/apache/atlas/util/SearchPredicateUtil.java
b/repository/src/main/java/org/apache/atlas/util/SearchPredicateUtil.java
index 9c2975645..bfc93087b 100644
--- a/repository/src/main/java/org/apache/atlas/util/SearchPredicateUtil.java
+++ b/repository/src/main/java/org/apache/atlas/util/SearchPredicateUtil.java
@@ -22,6 +22,7 @@ import org.apache.atlas.repository.graphdb.AtlasElement;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasRelationshipType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.Predicate;
@@ -1539,4 +1540,34 @@ public class SearchPredicateUtil {
}
+ public static Predicate
generateIsRelationshipEdgePredicate(AtlasTypeRegistry typeRegistry) {
+ return new IsRelationshipEdgePredicate(typeRegistry);
+ }
+
+ static class IsRelationshipEdgePredicate implements Predicate {
+ final AtlasTypeRegistry typeRegistry;
+
+
+ public IsRelationshipEdgePredicate(AtlasTypeRegistry typeRegistry) {
+ this.typeRegistry = typeRegistry;
+ }
+
+ @Override
+ public boolean evaluate(final Object object) {
+ final boolean ret;
+
+ AtlasEdge edge = (object instanceof AtlasEdge) ? (AtlasEdge)
object : null;
+
+ if (edge != null) {
+ String typeName =
AtlasGraphUtilsV2.getTypeName(edge);
+ AtlasRelationshipType type =
typeRegistry.getRelationshipTypeByName(typeName);
+
+ ret = type != null;
+ } else {
+ ret = false;
+ }
+
+ return ret;
+ }
+ }
}
diff --git a/repository/src/test/java/org/apache/atlas/BasicTestSetup.java
b/repository/src/test/java/org/apache/atlas/BasicTestSetup.java
index 9b0578fee..593f8c19b 100644
--- a/repository/src/test/java/org/apache/atlas/BasicTestSetup.java
+++ b/repository/src/test/java/org/apache/atlas/BasicTestSetup.java
@@ -28,9 +28,11 @@ import org.apache.atlas.model.instance.*;
import org.apache.atlas.model.typedef.*;
import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
+import org.apache.atlas.repository.store.graph.AtlasRelationshipStore;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.utils.TestLoadModelUtils;
import org.apache.commons.lang.StringUtils;
import javax.inject.Inject;
@@ -78,6 +80,8 @@ public abstract class BasicTestSetup extends AtlasTestBase {
@Inject
protected AtlasEntityStore entityStore;
@Inject
+ protected AtlasRelationshipStore relationshipStore;
+ @Inject
protected GlossaryService glossaryService;
private boolean baseLoaded = false;
@@ -94,6 +98,26 @@ public abstract class BasicTestSetup extends AtlasTestBase {
assignGlossary();
}
+ protected void setupRelationshipTestData() {
+ if(!baseLoaded) {
+ loadBaseModels();
+ }
+
+ try {
+
TestLoadModelUtils.loadModelFromResourcesJson("relationship_search_def.json",
typeDefStore, typeRegistry);
+ } catch (IOException e) {
+ fail("Failed to load relationship_search_def.json");
+ } catch (AtlasBaseException e) {
+ fail("Failed to load relationship_search_def.json to atlas model");
+ }
+
+ try {
+ createTestDataForRelationship();
+ } catch (AtlasBaseException e) {
+ fail("Failed to load data for relationship_search_def.json");
+ }
+ }
+
private void loadBaseModels() {
try {
loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore,
typeRegistry);
@@ -500,13 +524,14 @@ public abstract class BasicTestSetup extends
AtlasTestBase {
public SearchParameters.FilterCriteria getSingleFilterCondition(String
attName, SearchParameters.Operator op, String attrValue) {
SearchParameters.FilterCriteria filterCriteria = new
SearchParameters.FilterCriteria();
filterCriteria.setCondition(SearchParameters.FilterCriteria.Condition.AND);
+
List<SearchParameters.FilterCriteria> criteria = new ArrayList<>();
SearchParameters.FilterCriteria f1 = new
SearchParameters.FilterCriteria();
f1.setAttributeName(attName);
f1.setOperator(op);
- String time = String.valueOf(System.currentTimeMillis());
f1.setAttributeValue(attrValue);
criteria.add(f1);
+
filterCriteria.setCriterion(criteria);
return filterCriteria;
}
@@ -574,4 +599,164 @@ public abstract class BasicTestSetup extends
AtlasTestBase {
}
+ public static final String USER_TYPE = "user";
+ public static final String POST_TYPE = "post";
+ public static final String HIGHLIGHT_TYPE = "highlight";
+ public static final String USER_POST_TYPE = "user_post";
+ public static final String HIGHLIGHT_POST_TYPE = "highlight_post";
+
+ private void createTestDataForRelationship() throws AtlasBaseException {
+
+ List<AtlasEntity> entities = new ArrayList<>();
+
+ //users
+ AtlasEntity ajayUser = createUser("Ajay Mishra", new
Date(1985,10,9), "Male", "Content Writer");
+ entities.add(ajayUser);
+ AtlasEntity maryUser = createUser("Mary Williams", new
Date(1998,2,19), "Female", "Architecture");
+ entities.add(maryUser);
+ AtlasEntity divyaUser = createUser("Divya Deshmukh", new
Date(1980,12,10), "Female", "Designer");
+ entities.add(divyaUser);
+
+ //posts
+ AtlasEntity indAugPost = createPost("ind-post@Ajay", "Image", new
Date(2021, 8, 15), "Independence Day Celebration");
+ entities.add(indAugPost);
+ AtlasEntity onamAugPost = createPost("onam-post@Ajay", "Image", new
Date(2021, 8, 8), "Onam with Family");
+ entities.add(onamAugPost);
+ AtlasEntity diwaliPost = createPost("diwali-post@Ajay", "Reel", new
Date(2020, 11, 14), "Diwali get together");
+ entities.add(diwaliPost);
+ AtlasEntity tripPost = createPost("trip-post@Ajay", "Video", new
Date(2021, 3, 10), "Kashmir a serene");
+ entities.add(tripPost);
+ AtlasEntity officeParty = createPost("officeparty-post@Ajay", "Video",
new Date(2021, 4, 4), "office party");
+ entities.add(officeParty);
+
+ AtlasEntity indPost = createPost("ind-post@Mary", "Video", new
Date(2021, 8, 15), "Independence Day Celebration");
+ entities.add(indPost);
+ AtlasEntity christPost = createPost("christmas-post@Mary", "Image",
new Date(2020, 12, 25), "Christmas Day");
+ entities.add(christPost);
+ AtlasEntity sraPost = createPost("elevationSRA@Mary", "Image", new
Date(2020, 12, 25), "Elevation");
+ entities.add(sraPost);
+
+ AtlasEntity ganeshPost = createPost("ganeshchaturthi-post@Divya",
"Reel", new Date(2020, 12, 25), "Ganesh chaturthi");
+ entities.add(ganeshPost);
+
+ //highlights
+ AtlasEntity year2021 = createHighlight("year2021@Ajay", "Journey of
2021", new Date(2021, 9, 25));
+ entities.add(year2021);
+ AtlasEntity festival = createHighlight("festives@Ajay", "Festive
Celebration", new Date(2021, 9, 25));
+ entities.add(festival);
+
+ AtlasEntity projectSRA = createHighlight("projectSRA@Mary", "Project
SRA", new Date(2021, 9, 25));
+ entities.add(projectSRA);
+
+
+ // create entity types
+ EntityMutationResponse response = entityStore.createOrUpdate(new
AtlasEntityStream(entities), false);
+ List<AtlasEntityHeader> headers = response.getCreatedEntities();
+ setGuidInHeaders(entities, headers);
+
+ // create relationship types
+ createUserPostRelationship(ajayUser, indAugPost, "create");
+ createUserPostRelationship(ajayUser, onamAugPost, "create");
+ createUserPostRelationship(ajayUser, diwaliPost, "create");
+ createUserPostRelationship(ajayUser, tripPost, "create");
+ createUserPostRelationship(ajayUser, officeParty, "create");
+ createUserPostRelationship(ajayUser, christPost, "like");
+ createUserPostRelationship(ajayUser, ganeshPost, "wow");
+
+ createUserPostRelationship(maryUser, indPost, "create");
+ createUserPostRelationship(maryUser, christPost, "create");
+ createUserPostRelationship(maryUser, sraPost, "create");
+ createUserPostRelationship(maryUser, onamAugPost, "wow");
+
+ createUserPostRelationship(divyaUser, ganeshPost, "create");
+ createUserPostRelationship(divyaUser, indAugPost, "like");
+ createUserPostRelationship(divyaUser, indPost, "like");
+ createUserPostRelationship(divyaUser, onamAugPost,"love");
+ createUserPostRelationship(divyaUser, diwaliPost, "love");
+ createUserPostRelationship(divyaUser, christPost, "like");
+
+ createHighlightPostRelationship(year2021, indAugPost);
+ createHighlightPostRelationship(year2021, onamAugPost);
+ createHighlightPostRelationship(year2021, tripPost);
+ createHighlightPostRelationship(year2021, officeParty);
+
+ createHighlightPostRelationship(festival, indAugPost);
+ createHighlightPostRelationship(festival, onamAugPost);
+ createHighlightPostRelationship(festival, diwaliPost);
+
+ createHighlightPostRelationship(projectSRA, sraPost);
+ }
+
+ private AtlasEntity createUser(String id, Date birthDate, String gender,
String bio) {
+ AtlasEntity user = new AtlasEntity(USER_TYPE);
+ user.setAttribute("id", id);
+ user.setAttribute("birthDate", birthDate);
+ user.setAttribute("gender", gender);
+ user.setAttribute("bio", bio);
+
+ return user;
+ }
+
+ private AtlasEntity createPost(String id, String type, Date postDate,
String caption) {
+ AtlasEntity post = new AtlasEntity(POST_TYPE);
+ post.setAttribute("type", type);
+ post.setAttribute("post_date", postDate);
+ post.setAttribute("caption", caption);
+ post.setAttribute("id", id);
+
+ return post;
+ }
+
+ private AtlasEntity createHighlight(String id, String title, Date
createdOn) {
+ AtlasEntity highlight = new AtlasEntity(HIGHLIGHT_TYPE);
+ highlight.setAttribute("created_on", createdOn);
+ highlight.setAttribute("title", title);
+ highlight.setAttribute("id", id);
+
+ return highlight;
+ }
+
+ private void createUserPostRelationship(AtlasEntity user, AtlasEntity
post, String reaction) {
+ Map<String, Object> attributes = new HashMap<>();
+ attributes.put("user_name", user.getAttribute("id"));
+ attributes.put("post_name", post.getAttribute("id"));
+ attributes.put("reaction", reaction);
+ try {
+ AtlasRelationship relationship = new
AtlasRelationship(USER_POST_TYPE, getAtlasObjectId(user),
getAtlasObjectId(post), attributes);
+ relationshipStore.create(relationship);
+ } catch (AtlasBaseException e) {
+ fail("Failed to create Relationship between , " +
user.getTypeName() + ", " + post.getTypeName());
+ }
+ }
+
+ private void createHighlightPostRelationship(AtlasEntity highlight,
AtlasEntity post) {
+ Map<String, Object> attributes = new HashMap<>();
+ attributes.put("highlight_name", highlight.getAttribute("id"));
+ attributes.put("post_name", post.getAttribute("id"));
+ try {
+ AtlasRelationship relationship = new
AtlasRelationship(HIGHLIGHT_POST_TYPE, getAtlasObjectId(highlight),
getAtlasObjectId(post), attributes);
+ relationshipStore.create(relationship);
+ } catch (AtlasBaseException e) {
+ fail("Failed to create Relationship between , " +
highlight.getTypeName() + ", " + post.getTypeName());
+ }
+ }
+
+ private List<AtlasEntity> setGuidInHeaders(List<AtlasEntity> entities,
List<AtlasEntityHeader> createdEntities) {
+ List<String> assignedGuids = new ArrayList<>();
+
+ for (AtlasEntityHeader header : createdEntities) {
+ String id = header.getAttribute("id").toString();
+
+ for (AtlasEntity entity : entities) {
+ if (id.equalsIgnoreCase(entity.getAttribute("id").toString()))
{
+ String guid = header.getGuid();
+ entity.setGuid(guid);
+ assignedGuids.add(guid);
+
+ break;
+ }
+ }
+ }
+ return entities;
+ }
}
diff --git
a/repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java
b/repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java
index ecb398a44..282a3fd4a 100644
---
a/repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java
+++
b/repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java
@@ -26,6 +26,7 @@ import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.AtlasQuickSearchResult;
import org.apache.atlas.model.discovery.AtlasSearchResult;
import org.apache.atlas.model.discovery.QuickSearchParameters;
+import org.apache.atlas.model.discovery.RelationshipSearchParameters;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.discovery.AtlasAggregationEntry;
import org.apache.atlas.model.instance.AtlasClassification;
@@ -67,6 +68,7 @@ public class AtlasDiscoveryServiceTest extends BasicTestSetup
{
setupTestData();
createDimensionalTaggedEntity("sales");
createSpecialCharTestEntities();
+ setupRelationshipTestData();
}
/* TermSearchProcessor(TSP),
@@ -1033,6 +1035,17 @@ public class AtlasDiscoveryServiceTest extends
BasicTestSetup {
discoveryService.searchWithParameters(params);
}
+ @Test
+ public void searchRelations() throws AtlasBaseException {
+ RelationshipSearchParameters sp = new RelationshipSearchParameters();
+ sp.setRelationshipName("highlight_post");
+ sp.setMarker("*");
+ sp.setLimit(10);
+ sp.setRelationshipFilters(getSingleFilterCondition("highlight_name",
Operator.EQ, "year2021@Ajay"));
+
+ AtlasSearchResult sr =
discoveryService.searchRelationsWithParameters(sp);
+ assertEquals(sr.getRelations().size(), 4);
+ }
private String gethiveTableSalesFactGuid() throws AtlasBaseException {
if (salesFactGuid == null) {
diff --git
a/repository/src/test/java/org/apache/atlas/discovery/RelationshipSearchProcessorTest.java
b/repository/src/test/java/org/apache/atlas/discovery/RelationshipSearchProcessorTest.java
new file mode 100644
index 000000000..3b2b59863
--- /dev/null
+++
b/repository/src/test/java/org/apache/atlas/discovery/RelationshipSearchProcessorTest.java
@@ -0,0 +1,200 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.discovery;
+
+import org.apache.atlas.BasicTestSetup;
+import org.apache.atlas.SortOrder;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.repository.graph.AtlasGraphProvider;
+import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
+import org.apache.atlas.repository.graphdb.AtlasEdge;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.commons.collections.CollectionUtils;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import javax.inject.Inject;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class RelationshipSearchProcessorTest extends BasicTestSetup {
+ @Inject
+ private AtlasGraph graph;
+
+ @BeforeClass
+ public void setup() throws Exception {
+ super.initialize();
+
+ setupRelationshipTestData();
+ }
+
+ @Inject
+ public GraphBackedSearchIndexer indexer;
+
+ @Test
+ public void totalRelationships() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setRelationshipName("user_post");
+ params.setLimit(20);
+
+ executeAndAssert(params,17);
+ }
+
+ @Test
+ public void searchByAttribute() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setRelationshipName("user_post");
+ params.setRelationshipFilters(getSingleFilterCondition("user_name",
SearchParameters.Operator.CONTAINS, "Ajay"));
+ params.setLimit(20);
+
+ executeAndAssert(params, 7);
+ }
+
+ @Test
+ public void searchByAttributes() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setRelationshipName("user_post");
+
+ SearchParameters.FilterCriteria filterCriteria = new
SearchParameters.FilterCriteria();
+
filterCriteria.setCondition(SearchParameters.FilterCriteria.Condition.AND);
+
+ List<SearchParameters.FilterCriteria> criteria = new ArrayList<>();
+ SearchParameters.FilterCriteria f1 = new
SearchParameters.FilterCriteria();
+ f1.setAttributeName("post_name");
+ f1.setOperator(SearchParameters.Operator.EQ);
+ f1.setAttributeValue("christmas-post@Mary");
+ criteria.add(f1);
+
+ SearchParameters.FilterCriteria f2 = new
SearchParameters.FilterCriteria();
+ f2.setAttributeName("reaction");
+ f2.setOperator(SearchParameters.Operator.EQ);
+ f2.setAttributeValue("like");
+ criteria.add(f2);
+
+ filterCriteria.setCriterion(criteria);
+
+ params.setRelationshipFilters(filterCriteria);
+ params.setLimit(20);
+
+ executeAndAssert(params, 2);
+ }
+
+ @Test(expectedExceptions = AtlasBaseException.class,
expectedExceptionsMessageRegExp = "Attribute name not found for type user_post")
+ public void invalidAttribute() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setRelationshipName("user_post");
+ params.setRelationshipFilters(getSingleFilterCondition("name",
SearchParameters.Operator.CONTAINS, "Ajay"));
+ params.setLimit(20);
+
+ executeAndAssert(params, 0);
+ }
+
+ public void sortByPostName() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setRelationshipName("user_post");
+ params.setRelationshipFilters(getSingleFilterCondition("user_name",
SearchParameters.Operator.CONTAINS, "Ajay"));
+ params.setLimit(20);
+ params.setSortBy("post_name");
+ params.setSortOrder(SortOrder.DESCENDING);
+
+ List<AtlasEdge> edges = executeAndAssert(params, 7);
+
+ //ideally it should be below, but as the field is of Text type, it
tokenizes the string Eg: "christmas-post Mary" and hence we cannot guarentee
sorting
+
//assertEquals("christmas-post@Mary",edges.get(0).getProperty("user_post.post_name",
String.class));
+
//assertEquals("trip-post@Ajay",edges.get(6).getProperty("user_post.post_name",
String.class));
+
+
assertEquals("ganeshchaturthi-post@Divya",edges.get(0).getProperty("user_post.post_name",
String.class));
+ }
+
+ @Test
+ public void searchBymultipleTypes() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setRelationshipName("user_post,highlight_post");
+ params.setLimit(30);
+
+ executeAndAssert(params,25);
+ }
+
+ @Test(expectedExceptions = AtlasBaseException.class,
expectedExceptionsMessageRegExp = "Attribute post_name not found for type
highlight_post")
+ public void searchBymultipleTypesFilter() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setRelationshipName("user_post,highlight_post");
+ params.setRelationshipFilters(getSingleFilterCondition("post_name",
SearchParameters.Operator.CONTAINS, "Ajay"));
+ params.setLimit(30);
+
+ executeAndAssert(params,0);
+ }
+
+ @Test
+ public void searchByTypeMarker() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setRelationshipName("user_post,highlight_post");
+ params.setLimit(10);
+
+ //1st page
+ params.setMarker("*");
+ SearchContext context1 = new SearchContext(params, typeRegistry,
graph, Collections.<String>emptySet());
+ RelationshipSearchProcessor processor1 = new
RelationshipSearchProcessor(context1, indexer.getEdgeIndexKeys());
+
+ List<AtlasEdge> result1 = processor1.executeEdges();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(result1));
+ assertEquals(result1.size(), 10);
+ assertNotNull(processor1.getNextMarker());
+
+ //2nd page
+ params.setMarker(processor1.getNextMarker());
+ SearchContext context2 = new SearchContext(params, typeRegistry,
graph, Collections.<String>emptySet());
+ RelationshipSearchProcessor processor2 = new
RelationshipSearchProcessor(context2, indexer.getEdgeIndexKeys());
+
+ List<AtlasEdge> result2 = processor2.executeEdges();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(result2));
+ assertEquals(result2.size(), 10);
+ assertNotNull(processor2.getNextMarker());
+ }
+
+ private List<AtlasEdge> executeAndAssert(SearchParameters params, int
expected) throws AtlasBaseException {
+ SearchContext context = new SearchContext(params, typeRegistry, graph,
Collections.<String>emptySet());
+ RelationshipSearchProcessor processor = new
RelationshipSearchProcessor(context, indexer.getEdgeIndexKeys());
+
+ List<AtlasEdge> result = processor.executeEdges();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(result));
+ assertEquals(result.size(), expected);
+
+ return result;
+ }
+
+ @AfterClass
+ public void teardown() throws Exception {
+ AtlasGraphProvider.cleanup();
+
+ super.cleanup();
+ }
+}
diff --git a/repository/src/test/resources/relationship_search_def.json
b/repository/src/test/resources/relationship_search_def.json
new file mode 100644
index 000000000..e2a4fe132
--- /dev/null
+++ b/repository/src/test/resources/relationship_search_def.json
@@ -0,0 +1,259 @@
+{
+ "enumDefs": [{
+ "name": "gender",
+ "description": "Defines user's gender",
+ "typeVersion": "1.0",
+ "elementDefs": [{
+ "ordinal": 0,
+ "value": "MALE"
+ },
+ {
+ "ordinal": 1,
+ "value": "FEMALE"
+ }
+ ]
+ }, {
+ "name": "post_type",
+ "description": "Represents a type of a Post",
+ "typeVersion": "1.0",
+ "elementDefs": [{
+ "ordinal": 0,
+ "value": "Image"
+ },
+ {
+ "ordinal": 1,
+ "value": "Video"
+ },
+ {
+ "ordinal": 2,
+ "value": "Text"
+ },
+ {
+ "ordinal": 3,
+ "value": "Reel"
+ }
+ ]
+ }, {
+ "name": "reaction_type",
+ "description": "Represents a type of reaction",
+ "typeVersion": "1.0",
+ "elementDefs": [{
+ "ordinal": 0,
+ "value": "MALE"
+ },
+ {
+ "ordinal": 1,
+ "value": "FEMALE"
+ }
+ ]
+ }],
+ "structDefs": [],
+ "classificationDefs": [],
+ "entityDefs": [{
+ "name": "Reference",
+ "superTypes": [],
+ "typeVersion": "1.0",
+ "attributeDefs": [{
+ "name": "id",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": true
+ }]
+ }, {
+ "name": "user",
+ "description": "Represent a intagram user",
+ "typeVersion": "1.0",
+ "superTypes": ["Reference"],
+ "attributeDefs": [
+ {
+ "name": "birthdate",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "gender",
+ "typeName": "gender",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "bio",
+ "description": "other details/caption of user",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ }, {
+ "name": "friend",
+ "description": "friend of user",
+ "typeVersion": "1.0",
+ "superTypes": ["Reference"]
+ },{
+ "name": "post",
+ "description": "A piece of writing, image, video, or other item
of content",
+ "typeVersion": "1.0",
+ "superTypes": ["Reference"],
+ "attributeDefs": [{
+ "name": "type",
+ "description": "Represents a type of a Post",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "post_date",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "caption",
+ "description": "A title or piece of text
appearing",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+
+ }, {
+ "name": "highlight",
+ "description": "A collection of posts",
+ "typeVersion": "1.0",
+ "superTypes": ["Reference"],
+ "attributeDefs": [{
+ "name": "created_on",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ }, {
+ "name": "title",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ }]
+ }],
+ "relationshipDefs": [{
+ "name": "user_post",
+ "typeVersion": "1.0",
+ "relationshipCategory": "AGGREGATION",
+ "relationshipLabel": "__user.posts",
+ "attributeDefs": [{
+ "name": "user_name",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },{
+ "name": "post_name",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ }, {
+ "name": "reaction",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ }],
+ "endDef1": {
+ "type": "user",
+ "name": "posts",
+ "isContainer": true,
+ "cardinality": "SET"
+ },
+ "endDef2": {
+ "type": "post",
+ "name": "user",
+ "cardinality": "SET"
+ },
+ "propagateTags": "NONE"
+ }, {
+ "name": "highlight_post",
+ "typeVersion": "1.0",
+ "relationshipCategory": "AGGREGATION",
+ "attributeDefs": [{
+ "name": "user_name",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },{
+ "name": "highlight_name",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ }],
+ "endDef1": {
+ "type": "highlight",
+ "name": "posts",
+ "isContainer": true,
+ "cardinality": "SET"
+ },
+ "endDef2": {
+ "type": "post",
+ "name": "highlight",
+ "isContainer": false,
+ "cardinality": "SINGLE"
+ },
+ "propagateTags": "NONE"
+ },{
+ "name": "user_highlight",
+ "typeVersion": "1.0",
+ "relationshipCategory": "AGGREGATION",
+ "attributeDefs": [{
+ "name": "user_name",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },{
+ "name": "highlight_name",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ }],
+ "endDef1": {
+ "type": "highlight",
+ "name": "posts",
+ "isContainer": true,
+ "cardinality": "SET"
+ },
+ "endDef2": {
+ "type": "post",
+ "name": "highlight",
+ "isContainer": false,
+ "cardinality": "SINGLE"
+ },
+ "propagateTags": "NONE"
+ }]
+}
\ No newline at end of file
diff --git a/test-tools/src/main/resources/solr/core-template/solrconfig.xml
b/test-tools/src/main/resources/solr/core-template/solrconfig.xml
index fa38e72b6..1550052b4 100644
--- a/test-tools/src/main/resources/solr/core-template/solrconfig.xml
+++ b/test-tools/src/main/resources/solr/core-template/solrconfig.xml
@@ -445,7 +445,7 @@
-->
<lst name="defaults">
<str name="defType">edismax</str>
- <str name="qf">35x_t 5j9_t 7wl_t a9x_t but_t dfp_l f0l_t i6d_l
iyt_l jr9_t kjp_s lc5_t m4l_s mx1_t ohx_t xz9_i 1151_t 12px_t 14at_l 15vp_t
1891_t 19tx_t 1bet_t 1czp_t 1ekl_t 1gxx_t 1iit_l 1k3p_t 1lol_t 1o1x_t 1qf9_t
1ssl_t 1udh_t 1wqt_t 4d8l_t 4pvp_s 4oat_s 4lxh_s 4kcl_s 4nid_t 4vet_t 4rgl_t
4t1h_l 4ttx_t 505h_t 4ykl_t 51qd_t 58ud_t 5edh_t 5j45_t 5gqt_t 5hj9_t 5fyd_t
5pfp_t 5on9_t 5wjp_l 5y4l_l 5q85_t 5vr9_t 66th_l 68ed_l 658l_t 63np_t 69z9_t
5zph_t 60hx_t 622t_t 6k91_t 6l1h_ [...]
+ <str name="qf">35x_t 5j9_t 7wl_t a9x_t but_t dfp_l f0l_t i6d_l
iyt_l jr9_t kjp_s lc5_t m4l_s mx1_t ohx_t xz9_i 1151_t 12px_t 14at_l 15vp_t
1891_t 19tx_t 1bet_t 1czp_t 1ekl_t 1gxx_t 1iit_l 1k3p_t 1lol_t 1o1x_t 1qf9_t
1ssl_t 1udh_t 1wqt_t 4eth_t 4rgl_s 4pvp_s 4nid_s 4lxh_s 4p39_t 4wzp_t 4t1h_t
4umd_l 4vet_t 51qd_t 505h_t 53b9_t 5af9_t 5fyd_t 5kp1_t 5ibp_t 5j45_t 5hj9_t
5r0l_t 5q85_t 5y4l_l 5zph_l 5rt1_t 5xc5_t 68ed_l 69z9_l 66th_t 658l_t 6bk5_t
61ad_t 622t_t 63np_t 6ltx_t 6mmd_ [...]
<str name="hl.fl">*</str>
<bool name="hl.requireFieldMatch">true</bool>
<bool name="lowercaseOperators">true</bool>
diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
index 0b044d4cf..a6ca04f3c 100644
--- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
+++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
@@ -355,6 +355,82 @@ public class DiscoveryREST {
}
}
+ /**
+ * Relationship search to search for relations(edges)
+ *
+ * @param parameters RelationshipSearchParameters
+ * @return Atlas search result
+ * @throws AtlasBaseException
+ * @HTTP 200 On successful search
+ */
+ @Path("relations")
+ @POST
+ @Timed
+ public AtlasSearchResult relationSearch(RelationshipSearchParameters
parameters) throws AtlasBaseException {
+ AtlasPerfTracer perf = null;
+
+ try {
+ if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
+ perf = AtlasPerfTracer.getPerfTracer(PERF_LOG,
"DiscoveryREST.relationSearch(" + parameters + ")");
+ }
+
+ if (parameters.getLimit() < 0 || parameters.getOffset() < 0) {
+ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST,
"Limit/offset should be non-negative");
+ }
+
+ if (StringUtils.isEmpty(parameters.getRelationshipName()) &&
!isEmpty(parameters.getRelationshipFilters())) {
+ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST,
"RelationshipFilters specified without Type name");
+ }
+
+ return discoveryService.searchRelationsWithParameters(parameters);
+ } finally {
+ AtlasPerfTracer.log(perf);
+ }
+ }
+
+ /**
+ * Relationship search to search for relations(edges)
+ *
+ * @param relationshipName AtlasRelationshipType name
+ * @param sortByAttribute sort the result using this attribute name,
default value is 'name'
+ * @param sortOrder sorting order
+ * @param limit limit the result set to only include the
specified number of entries
+ * @param offset start offset of the result set (useful for
pagination)
+ * @return Atlas search result
+ * @throws AtlasBaseException
+ * @HTTP 200 On successful search
+ */
+
+ @Path("/relations")
+ @GET
+ @Timed
+ public AtlasSearchResult relationSearch(@QueryParam("relationshipName")
String relationshipName,
+ @QueryParam("offset")
int offset,
+ @QueryParam("limit")
int limit,
+ @QueryParam("sortBy")
String sortByAttribute,
+ @QueryParam("sortOrder")
SortOrder sortOrder,
+ @QueryParam("marker")
String marker) throws AtlasBaseException {
+ AtlasPerfTracer perf = null;
+
+ try {
+ if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
+ perf = AtlasPerfTracer.getPerfTracer(PERF_LOG,
"DiscoveryREST.relationSearch(" + relationshipName +
+ ", " + sortByAttribute + ", " + sortOrder + ", " +
limit + ", " + offset + ", " + marker + ")");
+ }
+
+ RelationshipSearchParameters parameters = new
RelationshipSearchParameters();
+ parameters.setRelationshipName(relationshipName);
+ parameters.setSortBy(sortByAttribute);
+ parameters.setSortOrder(sortOrder);
+ parameters.setLimit(limit);
+ parameters.setOffset(offset);
+ parameters.setMarker(marker);
+ return discoveryService.searchRelationsWithParameters(parameters);
+
+ } finally {
+ AtlasPerfTracer.log(perf);
+ }
+ }
/**
* Relationship search to search for related entities satisfying the
search parameters
*
