This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 075f331 ATLAS-3036: Improve FullTextMapper performance during entity
retrieval
075f331 is described below
commit 075f331b74a08c818e7030183288a54bcdf35487
Author: Sarath Subramanian <ssubraman...@hortonworks.com>
AuthorDate: Sun Jun 16 12:34:21 2019 -0700
ATLAS-3036: Improve FullTextMapper performance during entity retrieval
(cherry picked from commit a08a5a39a83fcfee0965471a3a1b1c29279feea3)
(cherry picked from commit 7ff396af047c8ff829dbe5b8a4094177fb3e9686)
(cherry picked from commit 2fdbc851db06f7ebe96cc3fde4d8507d088e7ec6)
(cherry picked from commit 66c2964e05f8421d8dfb395f1f0cf61d1f33a8e4)
---
.../atlas/repository/graph/FullTextMapperV2.java | 8 +-
.../store/graph/v2/EntityGraphRetriever.java | 92 ++++++++++++++++------
2 files changed, 76 insertions(+), 24 deletions(-)
diff --git
a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
index 77cecff..ea90952 100644
---
a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
+++
b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
@@ -131,7 +131,7 @@ public class FullTextMapperV2 {
entity = entityWithExtInfo != null ?
entityWithExtInfo.getEntity() : null;
entityExtInfo = entityWithExtInfo;
} else {
- entity = getAndCacheEntity(guid);
+ entity = getAndCacheEntity(guid, false);
entityExtInfo = null;
}
@@ -256,11 +256,15 @@ public class FullTextMapperV2 {
}
private AtlasEntity getAndCacheEntity(String guid) throws
AtlasBaseException {
+ return getAndCacheEntity(guid, true);
+ }
+
+ private AtlasEntity getAndCacheEntity(String guid, boolean
includeReferences) throws AtlasBaseException {
RequestContext context = RequestContext.get();
AtlasEntity entity = context.getEntity(guid);
if (entity == null) {
- entity = entityGraphRetriever.toAtlasEntity(guid);
+ entity = entityGraphRetriever.toAtlasEntity(guid,
includeReferences);
if (entity != null) {
context.cache(entity);
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
index 14f9916..e69d280 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
@@ -44,6 +44,7 @@ import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasElement;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasArrayType;
+import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasRelationshipType;
@@ -155,6 +156,10 @@ public class EntityGraphRetriever {
this.ignoreRelationshipAttr = ignoreRelationshipAttr;
}
+ public AtlasEntity toAtlasEntity(String guid, boolean includeReferences)
throws AtlasBaseException {
+ return mapVertexToAtlasEntity(getEntityVertex(guid), null, false,
includeReferences);
+ }
+
public AtlasEntity toAtlasEntity(String guid) throws AtlasBaseException {
return toAtlasEntity(getEntityVertex(guid));
}
@@ -422,6 +427,10 @@ public class EntityGraphRetriever {
}
private AtlasEntity mapVertexToAtlasEntity(AtlasVertex entityVertex,
AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo) throws
AtlasBaseException {
+ return mapVertexToAtlasEntity(entityVertex, entityExtInfo,
isMinExtInfo, true);
+ }
+
+ private AtlasEntity mapVertexToAtlasEntity(AtlasVertex entityVertex,
AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo, boolean
includeReferences) throws AtlasBaseException {
String guid = GraphHelper.getGuid(entityVertex);
AtlasEntity entity = entityExtInfo != null ?
entityExtInfo.getEntity(guid) : null;
@@ -438,7 +447,7 @@ public class EntityGraphRetriever {
mapSystemAttributes(entityVertex, entity);
- mapAttributes(entityVertex, entity, entityExtInfo, isMinExtInfo);
+ mapAttributes(entityVertex, entity, entityExtInfo, isMinExtInfo,
includeReferences);
if (!ignoreRelationshipAttr) { // only map when really needed
mapRelationshipAttributes(entityVertex, entity, entityExtInfo,
isMinExtInfo);
@@ -588,6 +597,10 @@ public class EntityGraphRetriever {
}
private void mapAttributes(AtlasVertex entityVertex, AtlasStruct struct,
AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo) throws
AtlasBaseException {
+ mapAttributes(entityVertex, struct, entityExtInfo, isMinExtInfo, true);
+ }
+
+ private void mapAttributes(AtlasVertex entityVertex, AtlasStruct struct,
AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo, boolean
includeReferences) throws AtlasBaseException {
AtlasType objType = typeRegistry.getType(struct.getTypeName());
if (!(objType instanceof AtlasStructType)) {
@@ -597,7 +610,7 @@ public class EntityGraphRetriever {
AtlasStructType structType = (AtlasStructType) objType;
for (AtlasAttribute attribute :
structType.getAllAttributes().values()) {
- Object attrValue = mapVertexToAttribute(entityVertex, attribute,
entityExtInfo, isMinExtInfo);
+ Object attrValue = mapVertexToAttribute(entityVertex, attribute,
entityExtInfo, isMinExtInfo, includeReferences);
struct.setAttribute(attribute.getName(), attrValue);
}
@@ -714,6 +727,10 @@ public class EntityGraphRetriever {
}
private Object mapVertexToAttribute(AtlasVertex entityVertex,
AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean
isMinExtInfo) throws AtlasBaseException {
+ return mapVertexToAttribute(entityVertex, attribute, entityExtInfo,
isMinExtInfo, true);
+ }
+
+ private Object mapVertexToAttribute(AtlasVertex entityVertex,
AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean
isMinExtInfo, boolean includeReferences) throws AtlasBaseException {
Object ret = null;
AtlasType attrType = attribute.getAttributeType();
String edgeLabel = attribute.getRelationshipEdgeLabel();
@@ -735,25 +752,56 @@ public class EntityGraphRetriever {
ret = mapVertexToStruct(entityVertex, edgeLabel, null,
entityExtInfo, isMinExtInfo);
break;
case OBJECT_ID_TYPE:
- if(attribute.getAttributeDef().isSoftReferenced()) {
- ret = mapVertexToObjectIdForSoftRef(entityVertex,
attribute, entityExtInfo, isMinExtInfo);
+ if (includeReferences) {
+ ret = attribute.getAttributeDef().isSoftReferenced() ?
mapVertexToObjectIdForSoftRef(entityVertex, attribute, entityExtInfo,
isMinExtInfo) :
+
mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo,
isOwnedAttribute, edgeDirection, isMinExtInfo);
} else {
- ret = mapVertexToObjectId(entityVertex, edgeLabel,
null, entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo);
- }
+ ret = null;
+ }
break;
- case ARRAY:
- if(attribute.getAttributeDef().isSoftReferenced()) {
- ret = mapVertexToArrayForSoftRef(entityVertex, attribute,
entityExtInfo, isMinExtInfo);
+ case ARRAY: {
+ final boolean skipAttribute;
+
+ if (!includeReferences) {
+ AtlasType elementType = ((AtlasArrayType)
attrType).getElementType();
+
+ skipAttribute = (elementType instanceof AtlasObjectIdType
|| elementType instanceof AtlasEntityType);
+ } else {
+ skipAttribute = false;
+ }
+
+ if (skipAttribute) {
+ ret = null;
} else {
- ret = mapVertexToArray(entityVertex, entityExtInfo,
isOwnedAttribute, attribute, isMinExtInfo);
- }
+ if (attribute.getAttributeDef().isSoftReferenced()) {
+ ret = mapVertexToArrayForSoftRef(entityVertex,
attribute, entityExtInfo, isMinExtInfo);
+ } else {
+ ret = mapVertexToArray(entityVertex, entityExtInfo,
isOwnedAttribute, attribute, isMinExtInfo, includeReferences);
+ }
+ }
+ }
break;
- case MAP:
- if(attribute.getAttributeDef().isSoftReferenced()) {
- ret = mapVertexToMapForSoftRef(entityVertex, attribute,
entityExtInfo, isMinExtInfo);
+ case MAP: {
+ final boolean skipAttribute;
+
+ if (!includeReferences) {
+ AtlasType valueType = ((AtlasMapType)
attrType).getValueType();
+
+ skipAttribute = (valueType instanceof AtlasObjectIdType ||
valueType instanceof AtlasEntityType);
} else {
- ret = mapVertexToMap(entityVertex, entityExtInfo,
isOwnedAttribute, attribute, isMinExtInfo);
- }
+ skipAttribute = false;
+ }
+
+ if (skipAttribute) {
+ ret = null;
+ } else {
+ if (attribute.getAttributeDef().isSoftReferenced()) {
+ ret = mapVertexToMapForSoftRef(entityVertex,
attribute, entityExtInfo, isMinExtInfo);
+ } else {
+ ret = mapVertexToMap(entityVertex, entityExtInfo,
isOwnedAttribute, attribute, isMinExtInfo, includeReferences);
+ }
+ }
+ }
break;
case CLASSIFICATION:
// do nothing
@@ -841,7 +889,7 @@ public class EntityGraphRetriever {
}
private Map<String, Object> mapVertexToMap(AtlasVertex entityVertex,
AtlasEntityExtInfo entityExtInfo,
- boolean isOwnedAttribute,
AtlasAttribute attribute, final boolean isMinExtInfo) throws AtlasBaseException
{
+ boolean isOwnedAttribute,
AtlasAttribute attribute, final boolean isMinExtInfo, boolean
includeReferences) throws AtlasBaseException {
Map<String, Object> ret = null;
AtlasMapType mapType = (AtlasMapType)
attribute.getAttributeType();
@@ -861,7 +909,7 @@ public class EntityGraphRetriever {
String mapKey = entry.getKey();
Object keyValue = entry.getValue();
Object mapValue =
mapVertexToCollectionEntry(entityVertex, mapValueType, keyValue,
attribute.getRelationshipEdgeLabel(),
-
entityExtInfo, isOwnedAttribute, attribute.getRelationshipEdgeDirection(),
isMinExtInfo);
+
entityExtInfo, isOwnedAttribute, attribute.getRelationshipEdgeDirection(),
isMinExtInfo, includeReferences);
if (mapValue != null) {
ret.put(mapKey, mapValue);
}
@@ -875,7 +923,7 @@ public class EntityGraphRetriever {
}
private List<Object> mapVertexToArray(AtlasVertex entityVertex,
AtlasEntityExtInfo entityExtInfo,
- boolean isOwnedAttribute,
AtlasAttribute attribute, final boolean isMinExtInfo) throws AtlasBaseException
{
+ boolean isOwnedAttribute,
AtlasAttribute attribute, final boolean isMinExtInfo, boolean
includeReferences) throws AtlasBaseException {
AtlasArrayType arrayType = (AtlasArrayType)
attribute.getAttributeType();
AtlasType arrayElementType = arrayType.getElementType();
@@ -902,7 +950,7 @@ public class EntityGraphRetriever {
}
Object arrValue = mapVertexToCollectionEntry(entityVertex,
arrayElementType, element, edgeLabel,
- entityExtInfo,
isOwnedAttribute, edgeDirection, isMinExtInfo);
+ entityExtInfo,
isOwnedAttribute, edgeDirection, isMinExtInfo, includeReferences);
if (arrValue != null) {
arrValues.add(arrValue);
@@ -914,7 +962,7 @@ public class EntityGraphRetriever {
private Object mapVertexToCollectionEntry(AtlasVertex entityVertex,
AtlasType arrayElement, Object value,
String edgeLabel,
AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute,
- AtlasRelationshipEdgeDirection
edgeDirection, final boolean isMinExtInfo) throws AtlasBaseException {
+ AtlasRelationshipEdgeDirection
edgeDirection, final boolean isMinExtInfo, boolean includeReferences) throws
AtlasBaseException {
Object ret = null;
switch (arrayElement.getTypeCategory()) {
@@ -933,7 +981,7 @@ public class EntityGraphRetriever {
break;
case OBJECT_ID_TYPE:
- ret = mapVertexToObjectId(entityVertex, edgeLabel, (AtlasEdge)
value, entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo);
+ ret = includeReferences ? mapVertexToObjectId(entityVertex,
edgeLabel, (AtlasEdge) value, entityExtInfo, isOwnedAttribute, edgeDirection,
isMinExtInfo) : null;
break;
default:
