http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 7f24d5a..03a86f4 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -24,7 +24,6 @@ import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasException; import org.apache.atlas.SortOrder; import org.apache.atlas.annotation.GraphTransaction; -import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasFullTextResult; @@ -44,7 +43,6 @@ import org.apache.atlas.query.QueryParser; import org.apache.atlas.query.QueryProcessor; import org.apache.atlas.query.SelectExpressionHelper; import org.apache.atlas.repository.Constants; -import org.apache.atlas.repository.MetadataRepository; import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasGraph; @@ -70,9 +68,6 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; -import scala.Option; -import scala.util.Either; -import scala.util.parsing.combinator.Parsers.NoSuccess; import javax.inject.Inject; import javax.script.Bindings; @@ -104,7 +99,6 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { private static final String DEFAULT_SORT_ATTRIBUTE_NAME = "name"; private final AtlasGraph graph; - private final DefaultGraphPersistenceStrategy graphPersistenceStrategy; private final EntityGraphRetriever entityRetriever; private final AtlasGremlinQueryProvider gremlinQueryProvider; private final AtlasTypeRegistry typeRegistry; @@ -116,11 +110,10 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { private final UserProfileService userProfileService; @Inject - EntityDiscoveryService(MetadataRepository metadataRepository, AtlasTypeRegistry typeRegistry, + EntityDiscoveryService(AtlasTypeRegistry typeRegistry, AtlasGraph graph, GraphBackedSearchIndexer indexer, SearchTracker searchTracker, UserProfileService userProfileService) throws AtlasException { this.graph = graph; - this.graphPersistenceStrategy = new DefaultGraphPersistenceStrategy(metadataRepository); this.entityRetriever = new EntityGraphRetriever(typeRegistry); this.indexer = indexer; this.searchTracker = searchTracker; @@ -685,15 +678,14 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { private GremlinQuery toGremlinQuery(String query, int limit, int offset) throws AtlasBaseException { QueryParams params = validateSearchParams(limit, offset); - Either<NoSuccess, Expression> either = QueryParser.apply(query, params); + Expression expression = QueryParser.apply(query, params); - if (either.isLeft()) { + if (expression == null) { throw new AtlasBaseException(DISCOVERY_QUERY_FAILED, query); } - Expression expression = either.right().get(); Expression validExpression = QueryProcessor.validate(expression); - GremlinQuery gremlinQuery = new GremlinTranslator(validExpression, graphPersistenceStrategy).translate(); + GremlinQuery gremlinQuery = new GremlinTranslator(validExpression).translate(); if (LOG.isDebugEnabled()) { LOG.debug("Translated Gremlin Query: {}", gremlinQuery.queryStr()); @@ -730,9 +722,9 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { List<List<Object>> values = new ArrayList<>(); // extract select attributes from gremlin query - Option<SelectExpression> selectExpr = SelectExpressionHelper.extractSelectExpression(query.expr()); - if (selectExpr.isDefined()) { - List<AliasExpression> aliases = selectExpr.get().toJavaList(); + SelectExpression selectExpr = SelectExpressionHelper.extractSelectExpression(query.expr()); + if (selectExpr != null) { + List<AliasExpression> aliases = selectExpr.toJavaList(); if (CollectionUtils.isNotEmpty(aliases)) { for (AliasExpression alias : aliases) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 3ae41c8..ae45c5c 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,6 +23,7 @@ import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.annotation.GraphTransaction; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.lineage.AtlasLineageInfo; import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection; @@ -31,11 +32,17 @@ import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1; import org.apache.atlas.repository.store.graph.v1.EntityGraphRetriever; +import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.type.AtlasTypeUtil; import org.apache.atlas.util.AtlasGremlinQueryProvider; import org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery; +import org.apache.atlas.v1.model.lineage.SchemaResponse.SchemaDetails; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; import org.springframework.stereotype.Service; import javax.inject.Inject; @@ -45,21 +52,24 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; @Service public class EntityLineageService implements AtlasLineageService { - private static final String INPUT_PROCESS_EDGE = "__Process.inputs"; - private static final String OUTPUT_PROCESS_EDGE = "__Process.outputs"; + private static final String INPUT_PROCESS_EDGE = "__Process.inputs"; + private static final String OUTPUT_PROCESS_EDGE = "__Process.outputs"; private final AtlasGraph graph; private final AtlasGremlinQueryProvider gremlinQueryProvider; private final EntityGraphRetriever entityRetriever; + private final AtlasTypeRegistry atlasTypeRegistry; @Inject - EntityLineageService(AtlasTypeRegistry typeRegistry, AtlasGraph atlasGraph) throws DiscoveryException { - this.graph = atlasGraph; + EntityLineageService(AtlasTypeRegistry typeRegistry, AtlasGraph atlasGraph) { + this.graph = atlasGraph; this.gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE; - this.entityRetriever = new EntityGraphRetriever(typeRegistry); + this.entityRetriever = new EntityGraphRetriever(typeRegistry); + this.atlasTypeRegistry = typeRegistry; } @Override @@ -88,6 +98,53 @@ public class EntityLineageService implements AtlasLineageService { return lineageInfo; } + @Override + @GraphTransaction + public SchemaDetails getSchemaForHiveTableByName(final String datasetName) throws AtlasBaseException { + if (StringUtils.isEmpty(datasetName)) { + // TODO: Complete error handling here + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST); + } + + AtlasEntityType hive_table = atlasTypeRegistry.getEntityTypeByName("hive_table"); + + Map<String, Object> lookupAttributes = new HashMap<>(); + lookupAttributes.put("qualifiedName", datasetName); + String guid = AtlasGraphUtilsV1.getGuidByUniqueAttributes(hive_table, lookupAttributes); + + return getSchemaForHiveTableByGuid(guid); + } + + @Override + @GraphTransaction + public SchemaDetails getSchemaForHiveTableByGuid(final String guid) throws AtlasBaseException { + if (StringUtils.isEmpty(guid)) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST); + } + SchemaDetails ret = new SchemaDetails(); + AtlasEntityType hive_column = atlasTypeRegistry.getEntityTypeByName("hive_column"); + + ret.setDataType(AtlasTypeUtil.toClassTypeDefinition(hive_column)); + + AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(guid); + Map<String, AtlasEntity> referredEntities = entityWithExtInfo.getReferredEntities(); + + if (MapUtils.isNotEmpty(referredEntities)) { + List<Map<String, Object>> rows = referredEntities.entrySet() + .stream() + .filter(EntityLineageService::isHiveColumn) + .map(e -> AtlasTypeUtil.toMap(e.getValue())) + .collect(Collectors.toList()); + ret.setRows(rows); + } + + return ret; + } + + private static boolean isHiveColumn(Map.Entry<String, AtlasEntity> e) { + return StringUtils.equals("hive_column", e.getValue().getTypeName()); + } + private AtlasLineageInfo getLineageInfo(String guid, LineageDirection direction, int depth) throws AtlasBaseException { Map<String, AtlasEntityHeader> entities = new HashMap<>(); Set<LineageRelation> relations = new HashSet<>(); @@ -108,7 +165,7 @@ public class EntityLineageService implements AtlasLineageService { continue; } - AtlasEntityHeader entity = entityRetriever.toAtlasEntityHeader((AtlasVertex)vertex); + AtlasEntityHeader entity = entityRetriever.toAtlasEntityHeader((AtlasVertex) vertex); if (!entities.containsKey(entity.getGuid())) { entities.put(entity.getGuid(), entity); @@ -143,7 +200,7 @@ public class EntityLineageService implements AtlasLineageService { return ret; } - private String getLineageQuery(String entityGuid, LineageDirection direction, int depth) throws AtlasBaseException { + private String getLineageQuery(String entityGuid, LineageDirection direction, int depth) { String lineageQuery = null; if (direction.equals(LineageDirection.INPUT)) { @@ -169,18 +226,19 @@ public class EntityLineageService implements AtlasLineageService { } private boolean entityExists(String guid) { - boolean ret = false; + boolean ret = false; Iterator<AtlasVertex> results = graph.query() - .has(Constants.GUID_PROPERTY_KEY, guid) - .vertices().iterator(); + .has(Constants.GUID_PROPERTY_KEY, guid) + .vertices().iterator(); while (results.hasNext()) { AtlasVertex entityVertex = results.next(); List<String> superTypes = GraphHelper.getSuperTypeNames(entityVertex); - ret = (CollectionUtils.isNotEmpty(superTypes)) ? superTypes.contains(AtlasClient.DATA_SET_SUPER_TYPE) : false; + ret = (CollectionUtils.isNotEmpty(superTypes)) && superTypes.contains(AtlasClient.DATA_SET_SUPER_TYPE); } return ret; } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/repository/src/main/java/org/apache/atlas/discovery/SearchIndexer.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchIndexer.java b/repository/src/main/java/org/apache/atlas/discovery/SearchIndexer.java index 8e67e32..b35346e 100755 --- a/repository/src/main/java/org/apache/atlas/discovery/SearchIndexer.java +++ b/repository/src/main/java/org/apache/atlas/discovery/SearchIndexer.java @@ -18,11 +18,9 @@ package org.apache.atlas.discovery; -import org.apache.atlas.listener.TypesChangeListener; - /** * Interface for indexing types. */ -public interface SearchIndexer extends TypesChangeListener { +public interface SearchIndexer { } http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java b/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java deleted file mode 100755 index 9b0aa4c..0000000 --- a/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java +++ /dev/null @@ -1,292 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.discovery.graph; - -import java.util.List; - -import javax.inject.Inject; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.RequestContext; -import org.apache.atlas.groovy.GroovyExpression; -import org.apache.atlas.query.GraphPersistenceStrategies; -import org.apache.atlas.query.GraphPersistenceStrategies$class; -import org.apache.atlas.query.TypeUtils; -import org.apache.atlas.repository.Constants; -import org.apache.atlas.repository.MetadataRepository; -import org.apache.atlas.repository.RepositoryException; -import org.apache.atlas.repository.graph.GraphBackedMetadataRepository; -import org.apache.atlas.repository.graph.GraphHelper; -import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; -import org.apache.atlas.repository.graphdb.AtlasGraph; -import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.repository.graphdb.GremlinVersion; -import org.apache.atlas.typesystem.ITypedReferenceableInstance; -import org.apache.atlas.typesystem.ITypedStruct; -import org.apache.atlas.typesystem.persistence.Id; -import org.apache.atlas.typesystem.types.AttributeInfo; -import org.apache.atlas.typesystem.types.ClassType; -import org.apache.atlas.typesystem.types.DataTypes; -import org.apache.atlas.typesystem.types.IDataType; -import org.apache.atlas.typesystem.types.Multiplicity; -import org.apache.atlas.typesystem.types.StructType; -import org.apache.atlas.typesystem.types.TraitType; -import org.apache.atlas.typesystem.types.TypeSystem; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.ImmutableCollection; -import com.google.common.collect.ImmutableList; - -/** - * Default implementation of GraphPersistenceStrategy. - */ -public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategies { - - private static final Logger LOG = LoggerFactory.getLogger(DefaultGraphPersistenceStrategy.class); - - private final GraphBackedMetadataRepository metadataRepository; - - @Inject - public DefaultGraphPersistenceStrategy(MetadataRepository metadataRepository) { - this.metadataRepository = (GraphBackedMetadataRepository) metadataRepository; - } - - @Override - public String typeAttributeName() { - return metadataRepository.getTypeAttributeName(); - } - - @Override - public String superTypeAttributeName() { - return metadataRepository.getSuperTypeAttributeName(); - } - - @Override - public String edgeLabel(IDataType<?> dataType, AttributeInfo aInfo) { - try { - return metadataRepository.getEdgeLabel(dataType, aInfo); - } catch (AtlasException e) { - throw new RuntimeException(e); - } - } - - @Override - public String traitLabel(IDataType<?> dataType, String traitName) { - return metadataRepository.getTraitLabel(dataType, traitName); - } - - @Override - public String fieldNameInVertex(IDataType<?> dataType, AttributeInfo aInfo) { - try { - return metadataRepository.getFieldNameInVertex(dataType, aInfo); - } catch (AtlasException e) { - throw new RuntimeException(e); - } - } - - @Override - public List<String> traitNames(AtlasVertex AtlasVertex) { - return GraphHelper.getTraitNames(AtlasVertex); - } - - @Override - public Id getIdFromVertex(String dataTypeName, AtlasVertex vertex) { - return GraphHelper.getIdFromVertex(dataTypeName, vertex); - } - - @Override - public ITypedReferenceableInstance constructClassInstanceId(ClassType classType, Object value) { - try { - AtlasVertex classVertex = (AtlasVertex) value; - ITypedReferenceableInstance classInstance = classType.createInstance(GraphHelper.getIdFromVertex(classVertex), - new String[0]); - return classType.convert(classInstance, Multiplicity.OPTIONAL); - } catch (AtlasException e) { - LOG.error("error while constructing an instance", e); - } - return null; - } - - @Override - public <U> U constructInstance(IDataType<U> dataType, Object value) { - try { - switch (dataType.getTypeCategory()) { - case PRIMITIVE: - case ENUM: - return dataType.convert(value, Multiplicity.OPTIONAL); - case ARRAY: - DataTypes.ArrayType arrType = (DataTypes.ArrayType) dataType; - IDataType<?> elemType = arrType.getElemType(); - ImmutableCollection.Builder result = ImmutableList.builder(); - List list = (List) value; - for(Object listElement : list) { - Object collectionEntry = constructCollectionEntry(elemType, listElement); - if(collectionEntry != null) { - result.add(collectionEntry); - } - } - return (U)result.build(); - case MAP: - // todo - break; - - case STRUCT: - AtlasVertex structVertex = (AtlasVertex) value; - StructType structType = (StructType) dataType; - ITypedStruct structInstance = structType.createInstance(); - TypeSystem.IdType idType = TypeSystem.getInstance().getIdType(); - - if (dataType.getName().equals(idType.getName())) { - structInstance.set(idType.typeNameAttrName(), GraphHelper.getSingleValuedProperty(structVertex, typeAttributeName(), String.class)); - structInstance.set(idType.idAttrName(), GraphHelper.getSingleValuedProperty(structVertex, idAttributeName(), String.class)); - String stateValue = GraphHelper.getSingleValuedProperty(structVertex, stateAttributeName(), String.class); - if (stateValue != null) { - structInstance.set(idType.stateAttrName(), stateValue); - } - structInstance.set(idType.versionAttrName(), structVertex.getProperty(versionAttributeName(), Integer.class)); - } else { - metadataRepository.getGraphToInstanceMapper() - .mapVertexToInstance(structVertex, structInstance, structType.fieldMapping().fields); - } - return dataType.convert(structInstance, Multiplicity.OPTIONAL); - - case TRAIT: - AtlasVertex traitVertex = (AtlasVertex) value; - TraitType traitType = (TraitType) dataType; - ITypedStruct traitInstance = traitType.createInstance(); - // todo - this is not right, we should load the Instance associated with this - // trait. for now just loading the trait struct. - // metadataRepository.getGraphToInstanceMapper().mapVertexToTraitInstance( - // traitVertex, dataType.getName(), , traitType, traitInstance); - metadataRepository.getGraphToInstanceMapper() - .mapVertexToInstance(traitVertex, traitInstance, traitType.fieldMapping().fields); - break; - - case CLASS: - AtlasVertex classVertex = (AtlasVertex) value; - String guid = classVertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class); - // Check if the instance we need was previously loaded. - ITypedReferenceableInstance classInstance = RequestContext.get().getInstanceV1(guid); - if (classInstance == null) { - classInstance = metadataRepository.getGraphToInstanceMapper().mapGraphToTypedInstance(guid, classVertex); - } - return dataType.convert(classInstance, Multiplicity.OPTIONAL); - - default: - throw new UnsupportedOperationException("Load for type " + dataType + "is not supported"); - } - } catch (AtlasException e) { - LOG.error("error while constructing an instance", e); - } - - return null; - } - - public <U> U constructCollectionEntry(IDataType<U> elementType, Object value) throws AtlasException { - switch (elementType.getTypeCategory()) { - case PRIMITIVE: - case ENUM: - return constructInstance(elementType, value); - //The array values in case of STRUCT, CLASS contain the edgeId if the outgoing edge which links to the STRUCT, CLASS vertex referenced - case STRUCT: - case CLASS: - String edgeId = (String) value; - return (U) metadataRepository.getGraphToInstanceMapper().getReferredEntity(edgeId, elementType); - case ARRAY: - case MAP: - case TRAIT: - return null; - default: - throw new UnsupportedOperationException("Load for type " + elementType + " in collections is not supported"); - } - } - - @Override - public String edgeLabel(TypeUtils.FieldInfo fInfo) { - return fInfo.reverseDataType() == null ? edgeLabel(fInfo.dataType(), fInfo.attrInfo()) : - edgeLabel(fInfo.reverseDataType(), fInfo.attrInfo()); - } - - @Override - public AtlasEdgeDirection instanceToTraitEdgeDirection() { - return AtlasEdgeDirection.OUT; - } - - @Override - public AtlasEdgeDirection traitToInstanceEdgeDirection() { - return AtlasEdgeDirection.IN; - } - - @Override - public String idAttributeName() { - return metadataRepository.getIdAttributeName(); - } - - @Override - public String stateAttributeName() { - return metadataRepository.getStateAttributeName(); - } - - @Override - public String versionAttributeName() { - return metadataRepository.getVersionAttributeName(); - } - - @Override - public boolean collectTypeInstancesIntoVar() { - return GraphPersistenceStrategies$class.collectTypeInstancesIntoVar(this); - } - - @Override - public boolean filterBySubTypes() { - return GraphPersistenceStrategies$class.filterBySubTypes(this); - } - - @Override - public boolean addGraphVertexPrefix(scala.collection.Traversable<GroovyExpression> preStatements) { - return GraphPersistenceStrategies$class.addGraphVertexPrefix(this, preStatements); - } - - @Override - public GremlinVersion getSupportedGremlinVersion() { - return GraphPersistenceStrategies$class.getSupportedGremlinVersion(this); - } - - @Override - public GroovyExpression generatePersisentToLogicalConversionExpression(GroovyExpression expr, IDataType<?> t) { - return GraphPersistenceStrategies$class.generatePersisentToLogicalConversionExpression(this,expr, t); - } - - @Override - public GroovyExpression addInitialQueryCondition(GroovyExpression expr) { - return GraphPersistenceStrategies$class.addInitialQueryCondition(this, expr); - } - - @Override - public boolean isPropertyValueConversionNeeded(IDataType<?> t) { - return GraphPersistenceStrategies$class.isPropertyValueConversionNeeded(this, t); - } - - @Override - public AtlasGraph getGraph() throws RepositoryException { - return metadataRepository.getGraph(); - } - -} http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/repository/src/main/java/org/apache/atlas/discovery/graph/GraphBackedDiscoveryService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/graph/GraphBackedDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/graph/GraphBackedDiscoveryService.java deleted file mode 100755 index aed8659..0000000 --- a/repository/src/main/java/org/apache/atlas/discovery/graph/GraphBackedDiscoveryService.java +++ /dev/null @@ -1,269 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.discovery.graph; - -import org.apache.atlas.AtlasClient; -import org.apache.atlas.annotation.GraphTransaction; -import org.apache.atlas.discovery.DiscoveryException; -import org.apache.atlas.discovery.DiscoveryService; -import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.query.Expressions; -import org.apache.atlas.query.GremlinEvaluator; -import org.apache.atlas.query.GremlinQuery; -import org.apache.atlas.query.GremlinQueryResult; -import org.apache.atlas.query.GremlinTranslator; -import org.apache.atlas.query.QueryParams; -import org.apache.atlas.query.QueryParser; -import org.apache.atlas.query.QueryProcessor; -import org.apache.atlas.repository.Constants; -import org.apache.atlas.repository.MetadataRepository; -import org.apache.atlas.repository.graph.GraphHelper; -import org.apache.atlas.repository.graphdb.AtlasEdge; -import org.apache.atlas.repository.graphdb.AtlasGraph; -import org.apache.atlas.repository.graphdb.AtlasIndexQuery; -import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.util.CompiledQueryCacheKey; -import org.apache.atlas.util.NoopGremlinQuery; -import org.codehaus.jettison.json.JSONArray; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; -import scala.util.Either; -import scala.util.parsing.combinator.Parsers; - -import javax.inject.Inject; -import javax.inject.Singleton; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -/** - * Graph backed implementation of Search. - */ -@Singleton -@Component -public class GraphBackedDiscoveryService implements DiscoveryService { - - private static final Logger LOG = LoggerFactory.getLogger(GraphBackedDiscoveryService.class); - - private final AtlasGraph graph; - private final DefaultGraphPersistenceStrategy graphPersistenceStrategy; - - public final static String SCORE = "score"; - /** - * Where the vertex' internal gremlin id is stored in the Map produced by extractResult() - */ - public final static String GREMLIN_ID_KEY = "id"; - /** - * Where the id of an edge's incoming vertex is stored in the Map produced by extractResult() - */ - public final static String GREMLIN_INVERTEX_KEY = "inVertex"; - /** - * Where the id of an edge's outgoing vertex is stored in the Map produced by extractResult() - */ - public final static String GREMLIN_OUTVERTEX_KEY = "outVertex"; - /** - * Where an edge's label is stored in the Map produced by extractResult() - */ - public final static String GREMLIN_LABEL_KEY = "label"; - - @Inject - GraphBackedDiscoveryService(MetadataRepository metadataRepository, AtlasGraph atlasGraph) - throws DiscoveryException { - this.graph = atlasGraph; - this.graphPersistenceStrategy = new DefaultGraphPersistenceStrategy(metadataRepository); - } - - //For titan 0.5.4, refer to http://s3.thinkaurelius.com/docs/titan/0.5.4/index-backends.html for indexed query - //http://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query - // .html#query-string-syntax for query syntax - @Override - @GraphTransaction - public String searchByFullText(String query, QueryParams queryParams) throws DiscoveryException { - String graphQuery = String.format("v.\"%s\":(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY, query); - LOG.debug("Full text query: {}", graphQuery); - Iterator<AtlasIndexQuery.Result<?, ?>> results =graph.indexQuery(Constants.FULLTEXT_INDEX, graphQuery).vertices(); - JSONArray response = new JSONArray(); - - int index = 0; - while (results.hasNext() && index < queryParams.offset()) { - results.next(); - index++; - } - - while (results.hasNext() && response.length() < queryParams.limit()) { - - AtlasIndexQuery.Result<?,?> result = results.next(); - AtlasVertex<?,?> vertex = result.getVertex(); - - JSONObject row = new JSONObject(); - String guid = GraphHelper.getGuid(vertex); - if (guid != null) { //Filter non-class entities - try { - row.put("guid", guid); - row.put(AtlasClient.TYPENAME, GraphHelper.getTypeName(vertex)); - row.put(SCORE, result.getScore()); - } catch (JSONException e) { - LOG.error("Unable to create response", e); - throw new DiscoveryException("Unable to create response"); - } - - response.put(row); - } - } - return response.toString(); - } - - @Override - @GraphTransaction - public String searchByDSL(String dslQuery, QueryParams queryParams) throws DiscoveryException { - GremlinQueryResult queryResult = evaluate(dslQuery, queryParams); - return queryResult.toJson(); - } - - public GremlinQueryResult evaluate(String dslQuery, QueryParams queryParams) throws DiscoveryException { - if(LOG.isDebugEnabled()) { - LOG.debug("Executing dsl query={}", dslQuery); - } - try { - GremlinQuery gremlinQuery = parseAndTranslateDsl(dslQuery, queryParams); - if(gremlinQuery instanceof NoopGremlinQuery) { - return new GremlinQueryResult(dslQuery, ((NoopGremlinQuery)gremlinQuery).getDataType(), Collections.emptyList()); - } - - return new GremlinEvaluator(gremlinQuery, graphPersistenceStrategy, graph).evaluate(); - - } catch (Exception e) { // unable to catch ExpressionException - throw new DiscoveryException("Invalid expression : " + dslQuery, e); - } - } - - private GremlinQuery parseAndTranslateDsl(String dslQuery, QueryParams queryParams) throws DiscoveryException { - - CompiledQueryCacheKey entry = new CompiledQueryCacheKey(dslQuery, queryParams); - GremlinQuery gremlinQuery = QueryProcessor.compiledQueryCache().get(entry); - if(gremlinQuery == null) { - Expressions.Expression validatedExpression = parseQuery(dslQuery, queryParams); - - //If the final limit is 0, don't launch the query, return with 0 rows - if (validatedExpression instanceof Expressions.LimitExpression - && ((Integer)((Expressions.LimitExpression) validatedExpression).limit().rawValue()) == 0) { - gremlinQuery = new NoopGremlinQuery(validatedExpression.dataType()); - } - else { - gremlinQuery = new GremlinTranslator(validatedExpression, graphPersistenceStrategy).translate(); - if (LOG.isDebugEnabled()) { - LOG.debug("Query = {}", validatedExpression); - LOG.debug("Expression Tree = {}", validatedExpression.treeString()); - LOG.debug("Gremlin Query = {}", gremlinQuery.queryStr()); - } - } - QueryProcessor.compiledQueryCache().put(entry, gremlinQuery); - } - return gremlinQuery; - } - - private Expressions.Expression parseQuery(String dslQuery, QueryParams queryParams) throws DiscoveryException { - Either<Parsers.NoSuccess, Expressions.Expression> either = QueryParser.apply(dslQuery, queryParams); - if (either.isRight()) { - Expressions.Expression expression = either.right().get(); - Expressions.Expression validatedExpression = QueryProcessor.validate(expression); - return validatedExpression; - } else { - throw new DiscoveryException("Invalid expression : " + dslQuery + ". " + either.left()); - } - - } - - /** - * Assumes the User is familiar with the persistence structure of the Repository. - * The given query is run uninterpreted against the underlying Graph Store. - * The results are returned as a List of Rows. each row is a Map of Key,Value pairs. - * - * @param gremlinQuery query in gremlin dsl format - * @return List of Maps - * @throws org.apache.atlas.discovery.DiscoveryException - */ - @Override - @GraphTransaction - public List<Map<String, String>> searchByGremlin(String gremlinQuery) throws DiscoveryException { - LOG.debug("Executing gremlin query={}", gremlinQuery); - try { - Object o = graph.executeGremlinScript(gremlinQuery, false); - return extractResult(o); - } catch (AtlasBaseException e) { - throw new DiscoveryException(e); - } - } - - private List<Map<String, String>> extractResult(final Object o) throws DiscoveryException { - List<Map<String, String>> result = new ArrayList<>(); - if (o instanceof List) { - List l = (List) o; - - for (Object value : l) { - Map<String, String> oRow = new HashMap<>(); - if (value instanceof Map) { - @SuppressWarnings("unchecked") Map<Object, Object> iRow = (Map) value; - for (Map.Entry e : iRow.entrySet()) { - Object k = e.getKey(); - Object v = e.getValue(); - oRow.put(k.toString(), v.toString()); - } - } else if (value instanceof AtlasVertex) { - AtlasVertex<?,?> vertex = (AtlasVertex<?,?>)value; - for (String key : vertex.getPropertyKeys()) { - Object propertyValue = GraphHelper.getProperty(vertex, key); - if (propertyValue != null) { - oRow.put(key, propertyValue.toString()); - } - } - oRow.put(GREMLIN_ID_KEY, vertex.getId().toString()); - - } else if (value instanceof String) { - oRow.put("", value.toString()); - } else if(value instanceof AtlasEdge) { - AtlasEdge edge = (AtlasEdge) value; - oRow.put(GREMLIN_ID_KEY, edge.getId().toString()); - oRow.put(GREMLIN_LABEL_KEY, edge.getLabel()); - oRow.put(GREMLIN_INVERTEX_KEY, edge.getInVertex().getId().toString()); - oRow.put(GREMLIN_OUTVERTEX_KEY, edge.getOutVertex().getId().toString()); - for (String propertyKey : edge.getPropertyKeys()) { - oRow.put(propertyKey, GraphHelper.getProperty(edge, propertyKey).toString()); - } - } else { - throw new DiscoveryException(String.format("Cannot process result %s", String.valueOf(value))); - } - - result.add(oRow); - } - } - else { - result.add(new HashMap<String, String>() {{ - put("result", o.toString()); - }}); - } - return result; - } -} http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/repository/src/main/java/org/apache/atlas/gremlin/Gremlin2ExpressionFactory.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/gremlin/Gremlin2ExpressionFactory.java b/repository/src/main/java/org/apache/atlas/gremlin/Gremlin2ExpressionFactory.java deleted file mode 100644 index 27de0ed..0000000 --- a/repository/src/main/java/org/apache/atlas/gremlin/Gremlin2ExpressionFactory.java +++ /dev/null @@ -1,379 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.gremlin; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.groovy.AbstractFunctionExpression; -import org.apache.atlas.groovy.CastExpression; -import org.apache.atlas.groovy.ClosureExpression; -import org.apache.atlas.groovy.ComparisonExpression; -import org.apache.atlas.groovy.ComparisonExpression.ComparisonOperator; -import org.apache.atlas.groovy.ComparisonOperatorExpression; -import org.apache.atlas.groovy.FieldExpression; -import org.apache.atlas.groovy.FunctionCallExpression; -import org.apache.atlas.groovy.GroovyExpression; -import org.apache.atlas.groovy.IdentifierExpression; -import org.apache.atlas.groovy.ListExpression; -import org.apache.atlas.groovy.LiteralExpression; -import org.apache.atlas.groovy.LogicalExpression; -import org.apache.atlas.groovy.LogicalExpression.LogicalOperator; -import org.apache.atlas.groovy.RangeExpression; -import org.apache.atlas.groovy.TernaryOperatorExpression; -import org.apache.atlas.groovy.TraversalStepType; -import org.apache.atlas.query.GraphPersistenceStrategies; -import org.apache.atlas.query.TypeUtils.FieldInfo; -import org.apache.atlas.typesystem.types.IDataType; - - -/** - * Generates gremlin query expressions using Gremlin 2 syntax. - * - */ -public class Gremlin2ExpressionFactory extends GremlinExpressionFactory { - - private static final String LOOP_METHOD = "loop"; - private static final String CONTAINS = "contains"; - private static final String LOOP_COUNT_FIELD = "loops"; - private static final String PATH_FIELD = "path"; - private static final String ENABLE_PATH_METHOD = "enablePath"; - private static final String BACK_METHOD = "back"; - private static final String LAST_METHOD = "last"; - - @Override - public GroovyExpression generateLogicalExpression(GroovyExpression parent, String operator, List<GroovyExpression> operands) { - return new FunctionCallExpression(TraversalStepType.FILTER, parent, operator, operands); - } - - - @Override - public GroovyExpression generateBackReferenceExpression(GroovyExpression parent, boolean inSelect, String alias) { - if (inSelect && parent == null) { - return getFieldInSelect(); - } - else if (inSelect && parent != null) { - return parent; - } - else { - return new FunctionCallExpression(TraversalStepType.MAP_TO_ELEMENT, parent, BACK_METHOD, new LiteralExpression(alias)); - } - } - - @Override - public GroovyExpression getLoopExpressionParent(GroovyExpression inputQry) { - return inputQry; - } - - @Override - public GroovyExpression generateLoopExpression(GroovyExpression parent,GraphPersistenceStrategies s, IDataType dataType, GroovyExpression loopExpr, String alias, Integer times) { - - GroovyExpression emitExpr = generateLoopEmitExpression(s, dataType); - //note that in Gremlin 2 (unlike Gremlin 3), the parent is not explicitly used. It is incorporated - //in the loopExpr. - GroovyExpression whileFunction = null; - if(times != null) { - GroovyExpression loopsExpr = new FieldExpression(getItVariable(), LOOP_COUNT_FIELD); - GroovyExpression timesExpr = new LiteralExpression(times); - whileFunction = new ClosureExpression(new ComparisonExpression(loopsExpr, ComparisonOperator.LESS_THAN, timesExpr)); - } - else { - GroovyExpression pathExpr = new FieldExpression(getItVariable(),PATH_FIELD); - GroovyExpression itObjectExpr = getCurrentObjectExpression(); - GroovyExpression pathContainsExpr = new FunctionCallExpression(pathExpr, CONTAINS, itObjectExpr); - whileFunction = new ClosureExpression(new TernaryOperatorExpression(pathContainsExpr, LiteralExpression.FALSE, LiteralExpression.TRUE)); - } - GroovyExpression emitFunction = new ClosureExpression(emitExpr); - GroovyExpression loopCall = new FunctionCallExpression(TraversalStepType.BRANCH, loopExpr, LOOP_METHOD, new LiteralExpression(alias), whileFunction, emitFunction); - - return new FunctionCallExpression(TraversalStepType.SIDE_EFFECT, loopCall, ENABLE_PATH_METHOD); - } - - @Override - public GroovyExpression typeTestExpression(GraphPersistenceStrategies s, String typeName, GroovyExpression itRef) { - - GroovyExpression superTypeAttrExpr = new FieldExpression(itRef, s.superTypeAttributeName()); - GroovyExpression typeNameExpr = new LiteralExpression(typeName); - GroovyExpression isSuperTypeExpr = new FunctionCallExpression(superTypeAttrExpr, CONTAINS, typeNameExpr); - GroovyExpression superTypeMatchesExpr = new TernaryOperatorExpression(superTypeAttrExpr, isSuperTypeExpr, LiteralExpression.FALSE); - - GroovyExpression typeAttrExpr = new FieldExpression(itRef, s.typeAttributeName()); - GroovyExpression typeMatchesExpr = new ComparisonExpression(typeAttrExpr, ComparisonOperator.EQUALS, typeNameExpr); - return new LogicalExpression(typeMatchesExpr, LogicalOperator.OR, superTypeMatchesExpr); - - } - - @Override - public GroovyExpression generateSelectExpression(GroovyExpression parent, List<LiteralExpression> sourceNames, - List<GroovyExpression> srcExprs) { - - GroovyExpression srcNamesExpr = new ListExpression(sourceNames); - List<GroovyExpression> selectArgs = new ArrayList<>(); - selectArgs.add(srcNamesExpr); - for(GroovyExpression expr : srcExprs) { - selectArgs.add(new ClosureExpression(expr)); - } - return new FunctionCallExpression(TraversalStepType.MAP_TO_VALUE, parent, SELECT_METHOD, selectArgs); - } - - @Override - public GroovyExpression generateFieldExpression(GroovyExpression parent, FieldInfo fInfo, String propertyName, boolean inSelect) { - return new FieldExpression(parent, propertyName); - } - - @Override - public GroovyExpression generateHasExpression(GraphPersistenceStrategies s, GroovyExpression parent, String propertyName, String symbol, - GroovyExpression requiredValue, FieldInfo fInfo) throws AtlasException { - GroovyExpression op = gremlin2CompOp(symbol); - GroovyExpression propertyNameExpr = new LiteralExpression(propertyName); - return new FunctionCallExpression(TraversalStepType.FILTER, parent, HAS_METHOD, propertyNameExpr, op, requiredValue); - } - - @Override - public GroovyExpression generateLikeExpressionUsingFilter(GroovyExpression parent, String propertyName, GroovyExpression propertyValue) throws AtlasException { - GroovyExpression itExpr = getItVariable(); - GroovyExpression nameExpr = new FieldExpression(itExpr, propertyName); - GroovyExpression matchesExpr = new FunctionCallExpression(nameExpr, MATCHES, escapePropertyValue(propertyValue)); - GroovyExpression closureExpr = new ClosureExpression(matchesExpr); - - return new FunctionCallExpression(TraversalStepType.FILTER, parent, FILTER_METHOD, closureExpr); - } - - private GroovyExpression escapePropertyValue(GroovyExpression propertyValue) { - GroovyExpression ret = propertyValue; - - if (propertyValue instanceof LiteralExpression) { - LiteralExpression exp = (LiteralExpression) propertyValue; - - if (exp != null && exp.getValue() instanceof String) { - String stringValue = (String) exp.getValue(); - - // replace '*' with ".*", replace '?' with '.' - stringValue = stringValue.replaceAll("\\*", ".*") - .replaceAll("\\?", "."); - - ret = new LiteralExpression(stringValue); - } - } - - return ret; - } - - private GroovyExpression gremlin2CompOp(String op) throws AtlasException { - - GroovyExpression tExpr = new IdentifierExpression("T"); - if(op.equals("=")) { - return new FieldExpression(tExpr, "eq"); - } - if(op.equals("!=")) { - return new FieldExpression(tExpr, "neq"); - } - if(op.equals(">")) { - return new FieldExpression(tExpr, "gt"); - } - if(op.equals(">=")) { - return new FieldExpression(tExpr, "gte"); - } - if(op.equals("<")) { - return new FieldExpression(tExpr, "lt"); - } - if(op.equals("<=")) { - return new FieldExpression(tExpr, "lte"); - } - if(op.equals("in")) { - return new FieldExpression(tExpr, "in"); - } - throw new AtlasException("Comparison operator " + op + " not supported in Gremlin"); - } - - @Override - protected GroovyExpression initialExpression(GroovyExpression varExpr, GraphPersistenceStrategies s) { - return generateSeededTraversalExpresssion(false, varExpr); - } - - @Override - public GroovyExpression generateSeededTraversalExpresssion(boolean isMap, GroovyExpression varExpr) { - return new FunctionCallExpression(TraversalStepType.START, varExpr, "_"); - } - - @Override - public GroovyExpression generateRangeExpression(GroovyExpression parent, int startIndex, int endIndex) { - //treat as barrier step, since limits need to be applied globally (even though it - //is technically a filter step) - return new RangeExpression(TraversalStepType.BARRIER, parent, startIndex, endIndex); - } - - @Override - public boolean isRangeExpression(GroovyExpression expr) { - - return (expr instanceof RangeExpression); - } - - @Override - public int[] getRangeParameters(AbstractFunctionExpression expr) { - - if (isRangeExpression(expr)) { - RangeExpression rangeExpression = (RangeExpression) expr; - return new int[] {rangeExpression.getStartIndex(), rangeExpression.getEndIndex()}; - } - else { - return null; - } - } - - @Override - public void setRangeParameters(GroovyExpression expr, int startIndex, int endIndex) { - - if (isRangeExpression(expr)) { - RangeExpression rangeExpression = (RangeExpression) expr; - rangeExpression.setStartIndex(startIndex); - rangeExpression.setEndIndex(endIndex); - } - else { - throw new IllegalArgumentException(expr.getClass().getName() + " is not a valid range expression - must be an instance of " + RangeExpression.class.getName()); - } - - } - - @Override - public List<GroovyExpression> getOrderFieldParents() { - - GroovyExpression itExpr = getItVariable(); - List<GroovyExpression> result = new ArrayList<>(2); - result.add(new FieldExpression(itExpr, "a")); - result.add(new FieldExpression(itExpr, "b")); - return result; - } - - - @Override - public GroovyExpression generateOrderByExpression(GroovyExpression parent, List<GroovyExpression> translatedOrderBy, boolean isAscending) { - - GroovyExpression aPropertyExpr = translatedOrderBy.get(0); - GroovyExpression bPropertyExpr = translatedOrderBy.get(1); - - GroovyExpression aPropertyNotNull = new ComparisonExpression(aPropertyExpr, ComparisonOperator.NOT_EQUALS, LiteralExpression.NULL); - GroovyExpression bPropertyNotNull = new ComparisonExpression(bPropertyExpr, ComparisonOperator.NOT_EQUALS, LiteralExpression.NULL); - - GroovyExpression aCondition = new TernaryOperatorExpression(aPropertyNotNull, new FunctionCallExpression(aPropertyExpr,TO_LOWER_CASE_METHOD), aPropertyExpr); - GroovyExpression bCondition = new TernaryOperatorExpression(bPropertyNotNull, new FunctionCallExpression(bPropertyExpr,TO_LOWER_CASE_METHOD), bPropertyExpr); - - GroovyExpression comparisonFunction = null; - if(isAscending) { - comparisonFunction = new ComparisonOperatorExpression(aCondition, bCondition); - } - else { - comparisonFunction = new ComparisonOperatorExpression(bCondition, aCondition); - } - return new FunctionCallExpression(TraversalStepType.BARRIER, parent, ORDER_METHOD, new ClosureExpression(comparisonFunction)); - } - - - @Override - public GroovyExpression getAnonymousTraversalExpression() { - return new FunctionCallExpression(TraversalStepType.START, "_"); - } - - - - @Override - public GroovyExpression generateGroupByExpression(GroovyExpression parent, GroovyExpression groupByExpression, - GroovyExpression aggregationFunction) { - GroovyExpression groupByClosureExpr = new ClosureExpression(groupByExpression); - GroovyExpression itClosure = new ClosureExpression(getItVariable()); - GroovyExpression result = new FunctionCallExpression(TraversalStepType.BARRIER, parent, "groupBy", groupByClosureExpr, itClosure); - result = new FunctionCallExpression(TraversalStepType.SIDE_EFFECT, result, "cap"); - result = new FunctionCallExpression(TraversalStepType.END, result, "next"); - result = new FunctionCallExpression(result, "values"); - result = new FunctionCallExpression(result, "toList"); - - GroovyExpression aggregrationFunctionClosure = new ClosureExpression(aggregationFunction); - result = new FunctionCallExpression(result, "collect", aggregrationFunctionClosure); - return result; - } - - @Override - public GroovyExpression getFieldInSelect() { - return getItVariable(); - } - @Override - public GroovyExpression getGroupBySelectFieldParent() { - GroovyExpression itExpr = getItVariable(); - return new FunctionCallExpression(itExpr, LAST_METHOD); - } - - //assumes cast already performed - @Override - public GroovyExpression generateCountExpression(GroovyExpression itExpr) { - return new FunctionCallExpression(itExpr, "size"); - } - - @Override - public String getTraversalExpressionClass() { - return "GremlinPipeline"; - } - - - @Override - public boolean isSelectGeneratesMap(int aliasCount) { - //in Gremlin 2 select always generates a map - return true; - } - - @Override - public GroovyExpression generateMapExpression(GroovyExpression parent, ClosureExpression closureExpression) { - return new FunctionCallExpression(TraversalStepType.MAP_TO_ELEMENT, parent, "transform", closureExpression); - } - - @Override - public GroovyExpression generateGetSelectedValueExpression(LiteralExpression key, - GroovyExpression rowMap) { - rowMap = new CastExpression(rowMap, "Row"); - GroovyExpression getExpr = new FunctionCallExpression(rowMap, "getColumn", key); - return getExpr; - } - - @Override - public GroovyExpression getCurrentTraverserObject(GroovyExpression traverser) { - return traverser; - } - - public List<String> getAliasesRequiredByExpression(GroovyExpression expr) { - if(!(expr instanceof FunctionCallExpression)) { - return Collections.emptyList(); - } - FunctionCallExpression fc = (FunctionCallExpression)expr; - if(! fc.getFunctionName().equals(LOOP_METHOD)) { - return Collections.emptyList(); - } - LiteralExpression aliasName = (LiteralExpression)fc.getArguments().get(0); - return Collections.singletonList(aliasName.getValue().toString()); - } - - @Override - public boolean isRepeatExpression(GroovyExpression expr) { - if(!(expr instanceof FunctionCallExpression)) { - return false; - } - return ((FunctionCallExpression)expr).getFunctionName().equals(LOOP_METHOD); - } -} - http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/repository/src/main/java/org/apache/atlas/gremlin/Gremlin3ExpressionFactory.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/gremlin/Gremlin3ExpressionFactory.java b/repository/src/main/java/org/apache/atlas/gremlin/Gremlin3ExpressionFactory.java deleted file mode 100644 index b936695..0000000 --- a/repository/src/main/java/org/apache/atlas/gremlin/Gremlin3ExpressionFactory.java +++ /dev/null @@ -1,485 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.gremlin; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.groovy.AbstractFunctionExpression; -import org.apache.atlas.groovy.CastExpression; -import org.apache.atlas.groovy.ClosureExpression; -import org.apache.atlas.groovy.ComparisonExpression; -import org.apache.atlas.groovy.ComparisonExpression.ComparisonOperator; -import org.apache.atlas.groovy.ComparisonOperatorExpression; -import org.apache.atlas.groovy.FieldExpression; -import org.apache.atlas.groovy.FunctionCallExpression; -import org.apache.atlas.groovy.GroovyExpression; -import org.apache.atlas.groovy.IdentifierExpression; -import org.apache.atlas.groovy.LiteralExpression; -import org.apache.atlas.groovy.LogicalExpression; -import org.apache.atlas.groovy.LogicalExpression.LogicalOperator; -import org.apache.atlas.groovy.TernaryOperatorExpression; -import org.apache.atlas.groovy.TraversalStepType; -import org.apache.atlas.groovy.TypeCoersionExpression; -import org.apache.atlas.query.GraphPersistenceStrategies; -import org.apache.atlas.query.TypeUtils.FieldInfo; -import org.apache.atlas.repository.graph.AtlasGraphProvider; -import org.apache.atlas.repository.graphdb.AtlasGraph; -import org.apache.atlas.typesystem.types.AttributeInfo; -import org.apache.atlas.typesystem.types.IDataType; - -/** - * Generates gremlin query expressions using Gremlin 3 syntax. - * - */ -public class Gremlin3ExpressionFactory extends GremlinExpressionFactory { - - - - private static final String VERTEX_LIST_CLASS = "List<Vertex>"; - private static final String VERTEX_ARRAY_CLASS = "Vertex[]"; - private static final String OBJECT_ARRAY_CLASS = "Object[]"; - private static final String VERTEX_CLASS = "Vertex"; - private static final String FUNCTION_CLASS = "Function"; - - private static final String VALUE_METHOD = "value"; - private static final String IS_PRESENT_METHOD = "isPresent"; - private static final String MAP_METHOD = "map"; - private static final String VALUES_METHOD = "values"; - private static final String GET_METHOD = "get"; - private static final String OR_ELSE_METHOD = "orElse"; - private static final String PROPERTY_METHOD = "property"; - private static final String BY_METHOD = "by"; - private static final String EQ_METHOD = "eq"; - private static final String EMIT_METHOD = "emit"; - private static final String TIMES_METHOD = "times"; - private static final String REPEAT_METHOD = "repeat"; - private static final String RANGE_METHOD = "range"; - private static final String LAST_METHOD = "last"; - private static final String TO_STRING_METHOD = "toString"; - - private static final GroovyExpression EMPTY_STRING_EXPRESSION = new LiteralExpression(""); - - @Override - public GroovyExpression generateLogicalExpression(GroovyExpression parent, String operator, - List<GroovyExpression> operands) { - return new FunctionCallExpression(TraversalStepType.FILTER, parent, operator, operands); - } - - @Override - public GroovyExpression generateBackReferenceExpression(GroovyExpression parent, boolean inSelect, String alias) { - if (inSelect) { - return getFieldInSelect(); - } else { - return new FunctionCallExpression(TraversalStepType.MAP_TO_ELEMENT, parent, SELECT_METHOD, new LiteralExpression(alias)); - } - } - - @Override - public GroovyExpression typeTestExpression(GraphPersistenceStrategies s, String typeName, GroovyExpression itRef) { - LiteralExpression superTypeAttrExpr = new LiteralExpression(s.superTypeAttributeName()); - LiteralExpression typeNameExpr = new LiteralExpression(typeName); - LiteralExpression typeAttrExpr = new LiteralExpression(s.typeAttributeName()); - FunctionCallExpression result = new FunctionCallExpression(TraversalStepType.FILTER, HAS_METHOD, typeAttrExpr, new FunctionCallExpression(EQ_METHOD, typeNameExpr)); - result = new FunctionCallExpression(TraversalStepType.FILTER, result, "or"); - result = new FunctionCallExpression(TraversalStepType.FILTER, result, HAS_METHOD, superTypeAttrExpr, new FunctionCallExpression(EQ_METHOD, typeNameExpr)); - return result; - - } - - @Override - public GroovyExpression generateLoopExpression(GroovyExpression parent,GraphPersistenceStrategies s, IDataType dataType, GroovyExpression loopExpr, String alias, Integer times) { - - GroovyExpression emitExpr = generateLoopEmitExpression(s, dataType); - - GroovyExpression result = new FunctionCallExpression(TraversalStepType.BRANCH, parent, REPEAT_METHOD, loopExpr); - if (times != null) { - GroovyExpression timesExpr = new LiteralExpression(times); - result = new FunctionCallExpression(TraversalStepType.SIDE_EFFECT, result, TIMES_METHOD, timesExpr); - } - result = new FunctionCallExpression(TraversalStepType.SIDE_EFFECT, result, EMIT_METHOD, emitExpr); - return result; - - } - - @Override - public GroovyExpression getLoopExpressionParent(GroovyExpression inputQry) { - GroovyExpression curTraversal = getAnonymousTraversalStartExpression(); - return curTraversal; - } - - private IdentifierExpression getAnonymousTraversalStartExpression() { - return new IdentifierExpression(TraversalStepType.START, "__"); - } - - @Override - public GroovyExpression generateSelectExpression(GroovyExpression parent, List<LiteralExpression> sourceNames, - List<GroovyExpression> srcExprs) { - FunctionCallExpression result = new FunctionCallExpression(TraversalStepType.MAP_TO_VALUE, parent, SELECT_METHOD, sourceNames); - - for (GroovyExpression expr : srcExprs) { - GroovyExpression closure = new ClosureExpression(expr); - GroovyExpression castClosure = new TypeCoersionExpression(closure, FUNCTION_CLASS); - result = new FunctionCallExpression(TraversalStepType.SIDE_EFFECT, result, BY_METHOD, castClosure); - } - return result; - } - - @Override - public GroovyExpression generateFieldExpression(GroovyExpression parent, FieldInfo fInfo, - String propertyName, boolean inSelect) { - - AttributeInfo attrInfo = fInfo.attrInfo(); - IDataType attrType = attrInfo.dataType(); - GroovyExpression propertyNameExpr = new LiteralExpression(propertyName); - //Whether it is the user or shared graph does not matter here, since we're - //just getting the conversion expression. Ideally that would be moved someplace else. - AtlasGraph graph = AtlasGraphProvider.getGraphInstance(); - if (inSelect) { - - GroovyExpression expr = new FunctionCallExpression(parent, PROPERTY_METHOD, propertyNameExpr); - expr = new FunctionCallExpression(expr, OR_ELSE_METHOD, LiteralExpression.NULL); - return graph.generatePersisentToLogicalConversionExpression(expr, attrType); - } else { - - GroovyExpression unmapped = new FunctionCallExpression(TraversalStepType.FLAT_MAP_TO_VALUES, parent, VALUES_METHOD, propertyNameExpr); - if (graph.isPropertyValueConversionNeeded(attrType)) { - GroovyExpression toConvert = new FunctionCallExpression(getItVariable(), GET_METHOD); - - GroovyExpression conversionFunction = graph.generatePersisentToLogicalConversionExpression(toConvert, - attrType); - return new FunctionCallExpression(TraversalStepType.MAP_TO_VALUE, unmapped, MAP_METHOD, new ClosureExpression(conversionFunction)); - } else { - return unmapped; - } - - } - } - - private ComparisonOperator getGroovyOperator(String symbol) throws AtlasException { - - String toFind = symbol; - if (toFind.equals("=")) { - toFind = "=="; - } - return ComparisonOperator.lookup(toFind); - } - - private String getComparisonFunction(String op) throws AtlasException { - - if (op.equals("=")) { - return "eq"; - } - if (op.equals("!=")) { - return "neq"; - } - if (op.equals(">")) { - return "gt"; - } - if (op.equals(">=")) { - return "gte"; - } - if (op.equals("<")) { - return "lt"; - } - if (op.equals("<=")) { - return "lte"; - } - throw new AtlasException("Comparison operator " + op + " not supported in Gremlin"); - } - - @Override - public GroovyExpression generateHasExpression(GraphPersistenceStrategies s, GroovyExpression parent, - String propertyName, String symbol, GroovyExpression requiredValue, FieldInfo fInfo) throws AtlasException { - - AttributeInfo attrInfo = fInfo.attrInfo(); - IDataType attrType = attrInfo.dataType(); - GroovyExpression propertNameExpr = new LiteralExpression(propertyName); - if (s.isPropertyValueConversionNeeded(attrType)) { - // for some types, the logical value cannot be stored directly in - // the underlying graph, - // and conversion logic is needed to convert the persistent form of - // the value - // to the actual value. In cases like this, we generate a conversion - // expression to - // do this conversion and use the filter step to perform the - // comparsion in the gremlin query - GroovyExpression itExpr = getItVariable(); - GroovyExpression vertexExpr = new CastExpression(new FunctionCallExpression(itExpr, GET_METHOD), VERTEX_CLASS); - GroovyExpression propertyValueExpr = new FunctionCallExpression(vertexExpr, VALUE_METHOD, propertNameExpr); - GroovyExpression conversionExpr = s.generatePersisentToLogicalConversionExpression(propertyValueExpr, - attrType); - - GroovyExpression propertyIsPresentExpression = new FunctionCallExpression( - new FunctionCallExpression(vertexExpr, PROPERTY_METHOD, propertNameExpr), IS_PRESENT_METHOD); - GroovyExpression valueMatchesExpr = new ComparisonExpression(conversionExpr, getGroovyOperator(symbol), - requiredValue); - - GroovyExpression filterCondition = new LogicalExpression(propertyIsPresentExpression, LogicalOperator.AND, - valueMatchesExpr); - - GroovyExpression filterFunction = new ClosureExpression(filterCondition); - return new FunctionCallExpression(TraversalStepType.FILTER, parent, FILTER_METHOD, filterFunction); - } else { - GroovyExpression valueMatches = new FunctionCallExpression(getComparisonFunction(symbol), requiredValue); - return new FunctionCallExpression(TraversalStepType.FILTER, parent, HAS_METHOD, propertNameExpr, valueMatches); - } - } - - @Override - public GroovyExpression generateLikeExpressionUsingFilter(GroovyExpression parent, String propertyName, GroovyExpression propertyValue) throws AtlasException { - GroovyExpression vertexExpr = new FunctionCallExpression(getItVariable(), GET_METHOD); - GroovyExpression propertyValueExpr = new FunctionCallExpression(vertexExpr, VALUE_METHOD, new LiteralExpression(propertyName)); - GroovyExpression matchesExpr = new FunctionCallExpression(propertyValueExpr, MATCHES, escapePropertyValue(propertyValue)); - GroovyExpression closureExpr = new ClosureExpression(matchesExpr); - - return new FunctionCallExpression(TraversalStepType.FILTER, parent, FILTER_METHOD, closureExpr); - } - - private GroovyExpression escapePropertyValue(GroovyExpression propertyValue) { - GroovyExpression ret = propertyValue; - - if (propertyValue instanceof LiteralExpression) { - LiteralExpression exp = (LiteralExpression) propertyValue; - - if (exp != null && exp.getValue() instanceof String) { - String stringValue = (String) exp.getValue(); - - // replace '*' with ".*", replace '?' with '.' - stringValue = stringValue.replaceAll("\\*", ".*") - .replaceAll("\\?", "."); - - ret = new LiteralExpression(stringValue); - } - } - - return ret; - } - - @Override - protected GroovyExpression initialExpression(GroovyExpression varExpr, GraphPersistenceStrategies s) { - - // this bit of groovy magic converts the set of vertices in varName into - // a String containing the ids of all the vertices. This becomes the - // argument - // to g.V(). This is needed because Gremlin 3 does not support - // _() - // s"g.V(${varName}.collect{it.id()} as String[])" - - GroovyExpression gExpr = getGraphExpression(); - GroovyExpression varRefExpr = new TypeCoersionExpression(varExpr, OBJECT_ARRAY_CLASS); - GroovyExpression matchingVerticesExpr = new FunctionCallExpression(TraversalStepType.START, gExpr, V_METHOD, varRefExpr); - GroovyExpression isEmpty = new FunctionCallExpression(varExpr, "isEmpty"); - GroovyExpression emptyGraph = getEmptyTraversalExpression(); - - GroovyExpression expr = new TernaryOperatorExpression(isEmpty, emptyGraph, matchingVerticesExpr); - - return s.addInitialQueryCondition(expr); - } - - private GroovyExpression getEmptyTraversalExpression() { - GroovyExpression emptyGraph = new FunctionCallExpression(TraversalStepType.START, getGraphExpression(), V_METHOD, EMPTY_STRING_EXPRESSION); - return emptyGraph; - } - - @Override - public GroovyExpression generateRangeExpression(GroovyExpression parent, int startIndex, int endIndex) { - //treat as barrier step, since limits need to be applied globally (even though it - //is technically a filter step) - return new FunctionCallExpression(TraversalStepType.BARRIER, parent, RANGE_METHOD, new LiteralExpression(startIndex), new LiteralExpression(endIndex)); - } - - @Override - public boolean isRangeExpression(GroovyExpression expr) { - - return (expr instanceof FunctionCallExpression && ((FunctionCallExpression)expr).getFunctionName().equals(RANGE_METHOD)); - } - - @Override - public int[] getRangeParameters(AbstractFunctionExpression expr) { - - if (isRangeExpression(expr)) { - FunctionCallExpression rangeExpression = (FunctionCallExpression) expr; - List<GroovyExpression> arguments = rangeExpression.getArguments(); - int startIndex = (int)((LiteralExpression)arguments.get(0)).getValue(); - int endIndex = (int)((LiteralExpression)arguments.get(1)).getValue(); - return new int[]{startIndex, endIndex}; - } - else { - return null; - } - } - - @Override - public void setRangeParameters(GroovyExpression expr, int startIndex, int endIndex) { - - if (isRangeExpression(expr)) { - FunctionCallExpression rangeExpression = (FunctionCallExpression) expr; - rangeExpression.setArgument(0, new LiteralExpression(Integer.valueOf(startIndex))); - rangeExpression.setArgument(1, new LiteralExpression(Integer.valueOf(endIndex))); - } - else { - throw new IllegalArgumentException(expr + " is not a valid range expression"); - } - } - - @Override - public List<GroovyExpression> getOrderFieldParents() { - - List<GroovyExpression> result = new ArrayList<>(1); - result.add(null); - return result; - } - - @Override - public GroovyExpression generateOrderByExpression(GroovyExpression parent, List<GroovyExpression> translatedOrderBy, - boolean isAscending) { - - GroovyExpression orderByExpr = translatedOrderBy.get(0); - GroovyExpression orderByClosure = new ClosureExpression(orderByExpr); - GroovyExpression orderByClause = new TypeCoersionExpression(orderByClosure, FUNCTION_CLASS); - - GroovyExpression aExpr = new IdentifierExpression("a"); - GroovyExpression bExpr = new IdentifierExpression("b"); - - GroovyExpression aCompExpr = new FunctionCallExpression(new FunctionCallExpression(aExpr, TO_STRING_METHOD), TO_LOWER_CASE_METHOD); - GroovyExpression bCompExpr = new FunctionCallExpression(new FunctionCallExpression(bExpr, TO_STRING_METHOD), TO_LOWER_CASE_METHOD); - - GroovyExpression comparisonExpr = null; - if (isAscending) { - comparisonExpr = new ComparisonOperatorExpression(aCompExpr, bCompExpr); - } else { - comparisonExpr = new ComparisonOperatorExpression(bCompExpr, aCompExpr); - } - ClosureExpression comparisonFunction = new ClosureExpression(comparisonExpr, "a", "b"); - FunctionCallExpression orderCall = new FunctionCallExpression(TraversalStepType.BARRIER, parent, ORDER_METHOD); - return new FunctionCallExpression(TraversalStepType.SIDE_EFFECT, orderCall, BY_METHOD, orderByClause, comparisonFunction); - } - - @Override - public GroovyExpression getAnonymousTraversalExpression() { - return null; - } - - @Override - public GroovyExpression getFieldInSelect() { - // this logic is needed to remove extra results from - // what is emitted by repeat loops. Technically - // for queries that don't have a loop in them we could just use "it" - // the reason for this is that in repeat loops with an alias, - // although the alias gets set to the right value, for some - // reason the select actually includes all vertices that were traversed - // through in the loop. In these cases, we only want the last vertex - // traversed in the loop to be selected. The logic here handles that - // case by converting the result to a list and just selecting the - // last item from it. - - GroovyExpression itExpr = getItVariable(); - GroovyExpression expr1 = new TypeCoersionExpression(itExpr, VERTEX_ARRAY_CLASS); - GroovyExpression expr2 = new TypeCoersionExpression(expr1, VERTEX_LIST_CLASS); - - return new FunctionCallExpression(expr2, LAST_METHOD); - } - - @Override - public GroovyExpression generateGroupByExpression(GroovyExpression parent, GroovyExpression groupByExpression, - GroovyExpression aggregationFunction) { - - GroovyExpression result = new FunctionCallExpression(TraversalStepType.BARRIER, parent, "group"); - GroovyExpression groupByClosureExpr = new TypeCoersionExpression(new ClosureExpression(groupByExpression), "Function"); - result = new FunctionCallExpression(TraversalStepType.SIDE_EFFECT, result, "by", groupByClosureExpr); - result = new FunctionCallExpression(TraversalStepType.END, result, "toList"); - - GroovyExpression mapValuesClosure = new ClosureExpression(new FunctionCallExpression(new CastExpression(getItVariable(), "Map"), "values")); - - result = new FunctionCallExpression(result, "collect", mapValuesClosure); - - //when we call Map.values(), we end up with an extra list around the result. We remove this by calling toList().get(0). This - //leaves us with a list of lists containing the vertices that match each group. We then apply the aggregation functions - //specified in the select list to each of these inner lists. - - result = new FunctionCallExpression(result ,"toList"); - result = new FunctionCallExpression(result, "get", new LiteralExpression(0)); - - GroovyExpression aggregrationFunctionClosure = new ClosureExpression(aggregationFunction); - result = new FunctionCallExpression(result, "collect", aggregrationFunctionClosure); - return result; - } - - @Override - public GroovyExpression generateSeededTraversalExpresssion(boolean isMap, GroovyExpression valueCollection) { - GroovyExpression coersedExpression = new TypeCoersionExpression(valueCollection, isMap ? "Map[]" : "Vertex[]"); - if(isMap) { - - return new FunctionCallExpression(TraversalStepType.START, "__", coersedExpression); - } - else { - //We cannot always use an anonymous traversal because that breaks repeat steps - return new FunctionCallExpression(TraversalStepType.START, getEmptyTraversalExpression(), "inject", coersedExpression); - } - } - - @Override - public GroovyExpression getGroupBySelectFieldParent() { - return null; - } - - @Override - public String getTraversalExpressionClass() { - return "GraphTraversal"; - } - - @Override - public boolean isSelectGeneratesMap(int aliasCount) { - //in Gremlin 3, you only get a map if there is more than 1 alias. - return aliasCount > 1; - } - - @Override - public GroovyExpression generateMapExpression(GroovyExpression parent, ClosureExpression closureExpression) { - return new FunctionCallExpression(TraversalStepType.MAP_TO_ELEMENT, parent, "map", closureExpression); - } - - @Override - public GroovyExpression generateGetSelectedValueExpression(LiteralExpression key, - GroovyExpression rowMapExpr) { - rowMapExpr = new CastExpression(rowMapExpr, "Map"); - GroovyExpression getExpr = new FunctionCallExpression(rowMapExpr, "get", key); - return getExpr; - } - - @Override - public GroovyExpression getCurrentTraverserObject(GroovyExpression traverser) { - return new FunctionCallExpression(traverser, "get"); - } - - public List<String> getAliasesRequiredByExpression(GroovyExpression expr) { - return Collections.emptyList(); - } - - @Override - public boolean isRepeatExpression(GroovyExpression expr) { - if(!(expr instanceof FunctionCallExpression)) { - return false; - } - return ((FunctionCallExpression)expr).getFunctionName().equals(REPEAT_METHOD); - } -}
