http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d2d6ff7d/repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala ---------------------------------------------------------------------- diff --git a/repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala b/repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala index f774d97..3ccc53f 100755 --- a/repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala +++ b/repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala @@ -21,17 +21,20 @@ package org.apache.atlas.query import java.util import java.util.Date -import com.thinkaurelius.titan.core.TitanVertex -import com.tinkerpop.blueprints.{Vertex, Direction} -import org.apache.atlas.AtlasException +import scala.collection.JavaConversions.seqAsJavaList + import org.apache.atlas.query.Expressions.{ComparisonExpression, ExpressionException} import org.apache.atlas.query.TypeUtils.FieldInfo import org.apache.atlas.repository.graph.{GraphHelper, GraphBackedMetadataRepository} +import org.apache.atlas.repository.RepositoryException +import org.apache.atlas.repository.graphdb._ import org.apache.atlas.typesystem.persistence.Id import org.apache.atlas.typesystem.types.DataTypes._ +import org.apache.atlas.typesystem.persistence.Id import org.apache.atlas.typesystem.types._ import org.apache.atlas.typesystem.{ITypedInstance, ITypedReferenceableInstance} + import scala.collection.JavaConversions._ import scala.collection.mutable import scala.collection.mutable.ArrayBuffer @@ -40,12 +43,23 @@ import scala.collection.mutable.ArrayBuffer * Represents the Bridge between the QueryProcessor and the Graph Persistence scheme used. * Some of the behaviors captured are: * - how is type and id information stored in the Vertex that represents an [[ITypedReferenceableInstance]] - * - how are edges representing trait and attribute relationships labelled. + * - how are edges representing trait and attribute relationships labeled. * - how are attribute names mapped to Property Keys in Vertices. * * This is a work in progress. + * */ trait GraphPersistenceStrategies { + + @throws(classOf[RepositoryException]) + def getGraph() : AtlasGraph[_,_] + + def getSupportedGremlinVersion() : GremlinVersion = getGraph().getSupportedGremlinVersion; + def generatePersisentToLogicalConversionExpression(expr: String, t: IDataType[_]) : String = getGraph().generatePersisentToLogicalConversionExpression(expr, t); + def isPropertyValueConversionNeeded(attrType: IDataType[_]) : Boolean = getGraph().isPropertyValueConversionNeeded(attrType); + + def initialQueryCondition = if (getGraph().requiresInitialIndexedPredicate()) { s""".${getGraph().getInitialIndexedPredicate}""" } else {""}; + /** * Name of attribute used to store typeName in vertex */ @@ -93,7 +107,7 @@ trait GraphPersistenceStrategies { * @param v * @return */ - def traitNames(v: TitanVertex): java.util.List[String] + def traitNames(v: AtlasVertex[_,_]): java.util.List[String] def edgeLabel(fInfo: FieldInfo): String = fInfo match { case FieldInfo(dataType, aInfo, null, null) => edgeLabel(dataType, aInfo) @@ -101,7 +115,26 @@ trait GraphPersistenceStrategies { case FieldInfo(dataType, null, null, traitName) => traitLabel(dataType, traitName) } - def fieldPrefixInSelect: String + def fieldPrefixInSelect(): String = { + + if(getSupportedGremlinVersion() == GremlinVersion.THREE) { + //this logic is needed to remove extra results from + //what is emitted by repeat loops. Technically + //for queries that don't have a loop in them we could just use "it" + //the reason for this is that in repeat loops with an alias, + //although the alias gets set to the right value, for some + //reason the select actually includes all vertices that were traversed + //through in the loop. In these cases, we only want the last vertex + //traversed in the loop to be selected. The logic here handles that + //case by converting the result to a list and just selecting the + //last item from it. + "((it as Vertex[]) as List<Vertex>).last()" + } + else { + "it" + } + + } /** * extract the Id from a Vertex. @@ -109,11 +142,30 @@ trait GraphPersistenceStrategies { * @param v * @return */ - def getIdFromVertex(dataTypeNm: String, v: TitanVertex): Id + def getIdFromVertex(dataTypeNm: String, v: AtlasVertex[_,_]): Id def constructInstance[U](dataType: IDataType[U], v: java.lang.Object): U - def gremlinCompOp(op: ComparisonExpression) = op.symbol match { + def gremlinCompOp(op: ComparisonExpression) = { + if( getSupportedGremlinVersion() == GremlinVersion.TWO) { + gremlin2CompOp(op); + } + else { + gremlin3CompOp(op); + } + } + + def gremlinPrimitiveOp(op: ComparisonExpression) = op.symbol match { + case "=" => "==" + case "!=" => "!=" + case ">" => ">" + case ">=" => ">=" + case "<" => "<" + case "<=" => "<=" + case _ => throw new ExpressionException(op, "Comparison operator not supported in Gremlin") + } + + private def gremlin2CompOp(op: ComparisonExpression) = op.symbol match { case "=" => "T.eq" case "!=" => "T.neq" case ">" => "T.gt" @@ -123,6 +175,16 @@ trait GraphPersistenceStrategies { case _ => throw new ExpressionException(op, "Comparison operator not supported in Gremlin") } + private def gremlin3CompOp(op: ComparisonExpression) = op.symbol match { + case "=" => "eq" + case "!=" => "neq" + case ">" => "gt" + case ">=" => "gte" + case "<" => "lt" + case "<=" => "lte" + case _ => throw new ExpressionException(op, "Comparison operator not supported in Gremlin") + } + def loopObjectExpression(dataType: IDataType[_]) = { _typeTestExpression(dataType.getName, "it.object") } @@ -162,13 +224,28 @@ trait GraphPersistenceStrategies { Seq(s"""filter${_typeTestExpression(typeName, "it")}""") } - private def _typeTestExpression(typeName: String, itRef: String): String = { - s"""{(${itRef}.'${typeAttributeName}' == '${typeName}') | - |(${itRef}.'${superTypeAttributeName}' ? - |${itRef}.'${superTypeAttributeName}'.contains('${typeName}') : false)}""". - stripMargin.replace(System.getProperty("line.separator"), "") + /** + * type test expression that ends up in the emit clause in + * loop/repeat steps and a few other places + */ + private def _typeTestExpression(typeName: String, itRef: String): String = { + + if( getSupportedGremlinVersion() == GremlinVersion.TWO) { + s"""{(${itRef}.'${typeAttributeName}' == '${typeName}') | + | (${itRef}.'${superTypeAttributeName}' ? + | ${itRef}.'${superTypeAttributeName}'.contains('${typeName}') : false)}""". + stripMargin.replace(System.getProperty("line.separator"), "") + } + else { + //gremlin 3 + s"""has('${typeAttributeName}',eq('${typeName}')).or().has('${superTypeAttributeName}',eq('${typeName}'))""" + } } + private def propertyValueSet(vertexRef : String, attrName: String) : String = { + s"""org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils.set(${vertexRef}.values('${attrName})""" + } + private def typeTestExpressionMultiStep(typeName : String, intSeq : IntSequence) : Seq[String] = { val varName = s"_var_${intSeq.next}" @@ -176,11 +253,21 @@ trait GraphPersistenceStrategies { newSetVar(varName), fillVarWithTypeInstances(typeName, varName), fillVarWithSubTypeInstances(typeName, varName), - s"$varName._()" + if(getSupportedGremlinVersion() == GremlinVersion.TWO) { + s"$varName._()" + } + else { + //this bit of groovy magic converts the set of vertices in varName into + //a String containing the ids of all the vertices. This becomes the argument + //to g.V(). This is needed because Gremlin 3 does not support + // _() + //s"g.V(${varName}.collect{it.id()} as String[])" + s"g.V(${varName} as Object[])${initialQueryCondition}" + } ) } - private def newSetVar(varName : String) = s"$varName = [] as Set" + private def newSetVar(varName : String) = s"def $varName = [] as Set" private def fillVarWithTypeInstances(typeName : String, fillVar : String) = { s"""g.V().has("${typeAttributeName}", "${typeName}").fill($fillVar)""" @@ -188,30 +275,39 @@ trait GraphPersistenceStrategies { private def fillVarWithSubTypeInstances(typeName : String, fillVar : String) = { s"""g.V().has("${superTypeAttributeName}", "${typeName}").fill($fillVar)""" - } + } } -object GraphPersistenceStrategy1 extends GraphPersistenceStrategies { +import scala.language.existentials; +import org.apache.atlas.repository.RepositoryException +import org.apache.atlas.repository.RepositoryException +import org.apache.atlas.repository.RepositoryException +import org.apache.atlas.repository.RepositoryException + +case class GraphPersistenceStrategy1(g: AtlasGraph[_,_]) extends GraphPersistenceStrategies { + val typeAttributeName = "typeName" val superTypeAttributeName = "superTypeNames" val idAttributeName = "guid" val stateAttributeName = "state" + override def getGraph() : AtlasGraph[_,_] = { + return g; + } + def edgeLabel(dataType: IDataType[_], aInfo: AttributeInfo) = s"__${dataType.getName}.${aInfo.name}" def edgeLabel(propertyName: String) = s"__${propertyName}" - val fieldPrefixInSelect = "it" - - def traitLabel(cls: IDataType[_], traitName: String) = s"$traitName" + def traitLabel(cls: IDataType[_], traitName: String) = s"${cls.getName}.$traitName" def fieldNameInVertex(dataType: IDataType[_], aInfo: AttributeInfo) = GraphHelper.getQualifiedFieldName(dataType, aInfo.name) - def getIdFromVertex(dataTypeNm: String, v: TitanVertex): Id = + def getIdFromVertex(dataTypeNm: String, v: AtlasVertex[_,_]): Id = new Id(v.getId.toString, 0, dataTypeNm) - def traitNames(v: TitanVertex): java.util.List[String] = { - val s = v.getProperty[String]("traitNames") + def traitNames(v: AtlasVertex[_,_]): java.util.List[String] = { + val s = v.getProperty("traitNames", classOf[String]) if (s != null) { Seq[String](s.split(","): _*) } else { @@ -228,17 +324,17 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies { if dataType.getName == TypeSystem.getInstance().getIdType.getName => { val sType = dataType.asInstanceOf[StructType] val sInstance = sType.createInstance() - val tV = v.asInstanceOf[TitanVertex] + val tV = v.asInstanceOf[AtlasVertex[_,_]] sInstance.set(TypeSystem.getInstance().getIdType.typeNameAttrName, - tV.getProperty[java.lang.String](typeAttributeName)) + tV.getProperty(typeAttributeName, classOf[java.lang.String])) sInstance.set(TypeSystem.getInstance().getIdType.idAttrName, - tV.getProperty[java.lang.String](idAttributeName)) + tV.getProperty(idAttributeName, classOf[java.lang.String])) dataType.convert(sInstance, Multiplicity.OPTIONAL) } case DataTypes.TypeCategory.STRUCT => { val sType = dataType.asInstanceOf[StructType] val sInstance = sType.createInstance() - loadStructInstance(sType, sInstance, v.asInstanceOf[TitanVertex]) + loadStructInstance(sType, sInstance, v.asInstanceOf[AtlasVertex[_,_]]) dataType.convert(sInstance, Multiplicity.OPTIONAL) } case DataTypes.TypeCategory.TRAIT => { @@ -248,12 +344,12 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies { * this is not right, we should load the Instance associated with this trait. * for now just loading the trait struct. */ - loadStructInstance(tType, tInstance, v.asInstanceOf[TitanVertex]) + loadStructInstance(tType, tInstance, v.asInstanceOf[AtlasVertex[_,_]]) dataType.convert(tInstance, Multiplicity.OPTIONAL) } case DataTypes.TypeCategory.CLASS => { val cType = dataType.asInstanceOf[ClassType] - val cInstance = constructClassInstance(dataType.asInstanceOf[ClassType], v.asInstanceOf[TitanVertex]) + val cInstance = constructClassInstance(dataType.asInstanceOf[ClassType], v.asInstanceOf[AtlasVertex[_,_]]) dataType.convert(cInstance, Multiplicity.OPTIONAL) } case DataTypes.TypeCategory.ENUM => dataType.convert(v, Multiplicity.OPTIONAL) @@ -262,7 +358,7 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies { } def loadStructInstance(dataType: IConstructableType[_, _ <: ITypedInstance], - typInstance: ITypedInstance, v: TitanVertex): Unit = { + typInstance: ITypedInstance, v: AtlasVertex[_,_]): Unit = { import scala.collection.JavaConversions._ dataType.fieldMapping().fields.foreach { t => val fName = t._1 @@ -271,15 +367,15 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies { } } - def constructClassInstance(dataType: ClassType, v: TitanVertex): ITypedReferenceableInstance = { + def constructClassInstance(dataType: ClassType, v: AtlasVertex[_,_]): ITypedReferenceableInstance = { val id = getIdFromVertex(dataType.name, v) val tNms = traitNames(v) val cInstance = dataType.createInstance(id, tNms: _*) // load traits tNms.foreach { tNm => val tLabel = traitLabel(dataType, tNm) - val edges = v.getEdges(Direction.OUT, tLabel) - val tVertex = edges.iterator().next().getVertex(Direction.IN).asInstanceOf[TitanVertex] + val edges = v.getEdges(AtlasEdgeDirection.OUT, tLabel) + val tVertex = edges.iterator().next().getInVertex().asInstanceOf[AtlasVertex[_,_]] val tType = TypeSystem.getInstance().getDataType[TraitType](classOf[TraitType], tNm) val tInstance = cInstance.getTrait(tNm).asInstanceOf[ITypedInstance] loadStructInstance(tType, tInstance, tVertex) @@ -288,7 +384,7 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies { cInstance } - def loadAttribute(dataType: IDataType[_], aInfo: AttributeInfo, i: ITypedInstance, v: TitanVertex): Unit = { + def loadAttribute(dataType: IDataType[_], aInfo: AttributeInfo, i: ITypedInstance, v: AtlasVertex[_,_]): Unit = { aInfo.dataType.getTypeCategory match { case DataTypes.TypeCategory.PRIMITIVE => loadPrimitiveAttribute(dataType, aInfo, i, v) case DataTypes.TypeCategory.ENUM => loadEnumAttribute(dataType, aInfo, i, v) @@ -303,26 +399,26 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies { } } - private def loadEnumAttribute(dataType: IDataType[_], aInfo: AttributeInfo, i: ITypedInstance, v: TitanVertex) + private def loadEnumAttribute(dataType: IDataType[_], aInfo: AttributeInfo, i: ITypedInstance, v: AtlasVertex[_,_]) : Unit = { val fName = fieldNameInVertex(dataType, aInfo) - i.setInt(aInfo.name, v.getProperty[java.lang.Integer](fName)) + i.setInt(aInfo.name, v.getProperty(fName, classOf[java.lang.Integer])) } private def loadPrimitiveAttribute(dataType: IDataType[_], aInfo: AttributeInfo, - i: ITypedInstance, v: TitanVertex): Unit = { + i: ITypedInstance, v: AtlasVertex[_,_]): Unit = { val fName = fieldNameInVertex(dataType, aInfo) aInfo.dataType() match { - case x: BooleanType => i.setBoolean(aInfo.name, v.getProperty[java.lang.Boolean](fName)) - case x: ByteType => i.setByte(aInfo.name, v.getProperty[java.lang.Byte](fName)) - case x: ShortType => i.setShort(aInfo.name, v.getProperty[java.lang.Short](fName)) - case x: IntType => i.setInt(aInfo.name, v.getProperty[java.lang.Integer](fName)) - case x: LongType => i.setLong(aInfo.name, v.getProperty[java.lang.Long](fName)) - case x: FloatType => i.setFloat(aInfo.name, v.getProperty[java.lang.Float](fName)) - case x: DoubleType => i.setDouble(aInfo.name, v.getProperty[java.lang.Double](fName)) - case x: StringType => i.setString(aInfo.name, v.getProperty[java.lang.String](fName)) + case x: BooleanType => i.setBoolean(aInfo.name, v.getProperty(fName, classOf[java.lang.Boolean])) + case x: ByteType => i.setByte(aInfo.name, v.getProperty(fName, classOf[java.lang.Byte])) + case x: ShortType => i.setShort(aInfo.name, v.getProperty(fName, classOf[java.lang.Short])) + case x: IntType => i.setInt(aInfo.name, v.getProperty(fName, classOf[java.lang.Integer])) + case x: LongType => i.setLong(aInfo.name, v.getProperty(fName, classOf[java.lang.Long])) + case x: FloatType => i.setFloat(aInfo.name, v.getProperty(fName, classOf[java.lang.Float])) + case x: DoubleType => i.setDouble(aInfo.name, v.getProperty(fName, classOf[java.lang.Double])) + case x: StringType => i.setString(aInfo.name, v.getProperty(fName, classOf[java.lang.String])) case x: DateType => { - val dateVal = v.getProperty[java.lang.Long](fName) + val dateVal = v.getProperty(fName, classOf[java.lang.Long]) i.setDate(aInfo.name, new Date(dateVal)) } case _ => throw new UnsupportedOperationException(s"load for ${aInfo.dataType()} not supported") @@ -331,9 +427,9 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies { private def loadArrayAttribute[T](dataType: IDataType[_], aInfo: AttributeInfo, - i: ITypedInstance, v: TitanVertex): Unit = { + i: ITypedInstance, v: AtlasVertex[_,_]): Unit = { import scala.collection.JavaConversions._ - val list: java.util.List[_] = v.getProperty(aInfo.name) + val list: java.util.List[_] = v.getListProperty(aInfo.name) val arrayType: DataTypes.ArrayType = aInfo.dataType.asInstanceOf[ArrayType] var values = new util.ArrayList[Any] @@ -344,13 +440,13 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies { } private def loadStructAttribute(dataType: IDataType[_], aInfo: AttributeInfo, - i: ITypedInstance, v: TitanVertex, edgeLbl: Option[String] = None): Unit = { + i: ITypedInstance, v: AtlasVertex[_,_], edgeLbl: Option[String] = None): Unit = { val eLabel = edgeLbl match { case Some(x) => x case None => edgeLabel(FieldInfo(dataType, aInfo, null)) } - val edges = v.getEdges(Direction.OUT, eLabel) - val sVertex = edges.iterator().next().getVertex(Direction.IN).asInstanceOf[TitanVertex] + val edges = v.getEdges(AtlasEdgeDirection.OUT, eLabel) + val sVertex = edges.iterator().next().getInVertex().asInstanceOf[AtlasVertex[_,_]] if (aInfo.dataType().getTypeCategory == DataTypes.TypeCategory.STRUCT) { val sType = aInfo.dataType().asInstanceOf[StructType] val sInstance = sType.createInstance() @@ -364,7 +460,7 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies { - private def mapVertexToCollectionEntry(instanceVertex: TitanVertex, attributeInfo: AttributeInfo, elementType: IDataType[_], i: ITypedInstance, value: Any): Any = { + private def mapVertexToCollectionEntry(instanceVertex: AtlasVertex[_,_], attributeInfo: AttributeInfo, elementType: IDataType[_], i: ITypedInstance, value: Any): Any = { elementType.getTypeCategory match { case DataTypes.TypeCategory.PRIMITIVE => value case DataTypes.TypeCategory.ENUM => value
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d2d6ff7d/repository/src/main/scala/org/apache/atlas/query/GremlinEvaluator.scala ---------------------------------------------------------------------- diff --git a/repository/src/main/scala/org/apache/atlas/query/GremlinEvaluator.scala b/repository/src/main/scala/org/apache/atlas/query/GremlinEvaluator.scala index 10d66a9..6803036 100755 --- a/repository/src/main/scala/org/apache/atlas/query/GremlinEvaluator.scala +++ b/repository/src/main/scala/org/apache/atlas/query/GremlinEvaluator.scala @@ -18,10 +18,9 @@ package org.apache.atlas.query -import javax.script.{Bindings, ScriptEngine, ScriptEngineManager} + import org.apache.atlas.query.Expressions._ -import com.thinkaurelius.titan.core.TitanGraph -import com.tinkerpop.pipes.util.structures.Row +import org.apache.atlas.repository.graphdb.AtlasGraph import org.apache.atlas.query.TypeUtils.ResultWithPathStruct import org.apache.atlas.typesystem.json._ import org.apache.atlas.typesystem.types._ @@ -40,81 +39,109 @@ case class GremlinQueryResult(query: String, def toJson = JsonHelper.toJson(this) } -class GremlinEvaluator(qry: GremlinQuery, persistenceStrategy: GraphPersistenceStrategies, g: TitanGraph) { - - val manager: ScriptEngineManager = new ScriptEngineManager - val engine: ScriptEngine = manager.getEngineByName("gremlin-groovy") - val bindings: Bindings = engine.createBindings - bindings.put("g", g) +class GremlinEvaluator(qry: GremlinQuery, persistenceStrategy: GraphPersistenceStrategies, g: AtlasGraph[_,_]) { - /** + /** * * @param gResultObj is the object returned from gremlin. This must be a List * @param qryResultObj is the object constructed for the output w/o the Path. * @return a ResultWithPathStruct */ - def addPathStruct(gResultObj : AnyRef, qryResultObj : Any) : Any = { - if ( !qry.isPathExpresion) { - qryResultObj - } else { - import scala.collection.JavaConversions._ - import scala.collection.JavaConverters._ - val iPaths = gResultObj.asInstanceOf[java.util.List[AnyRef]].init - - val oPaths = iPaths.map { p => - persistenceStrategy.constructInstance(TypeSystem.getInstance().getIdType.getStructType, p) - }.toList.asJava - val sType = qry.expr.dataType.asInstanceOf[StructType] - val sInstance = sType.createInstance() - sInstance.set(ResultWithPathStruct.pathAttrName, oPaths) - sInstance.set(ResultWithPathStruct.resultAttrName, qryResultObj) - sInstance - } + def addPathStruct(gResultObj: AnyRef, qryResultObj: Any): Any = { + if (!qry.isPathExpression) { + qryResultObj + } else { + import scala.collection.JavaConversions._ + import scala.collection.JavaConverters._ + + val iPaths = gResultObj.asInstanceOf[java.util.List[AnyRef]].init + + val oPaths = iPaths.map { value => + persistenceStrategy.constructInstance(TypeSystem.getInstance().getIdType.getStructType, value) + }.toList.asJava + val sType = qry.expr.dataType.asInstanceOf[StructType] + val sInstance = sType.createInstance() + sInstance.set(ResultWithPathStruct.pathAttrName, oPaths) + sInstance.set(ResultWithPathStruct.resultAttrName, qryResultObj) + sInstance + } } - def instanceObject(v : AnyRef) : AnyRef = { - if ( qry.isPathExpresion ) { - import scala.collection.JavaConversions._ - v.asInstanceOf[java.util.List[AnyRef]].last - } else { - v - } + def instanceObject(v: AnyRef): AnyRef = { + if (qry.isPathExpression) { + import scala.collection.JavaConversions._ + v.asInstanceOf[java.util.List[AnyRef]].last + } else { + v + } } def evaluate(): GremlinQueryResult = { import scala.collection.JavaConversions._ + val debug:Boolean = false val rType = qry.expr.dataType - val oType = if (qry.isPathExpresion) qry.expr.children(0).dataType else rType - val rawRes = engine.eval(qry.queryStr, bindings) + val oType = if (qry.isPathExpression) { + qry.expr.children(0).dataType + } + else { + rType + } + val rawRes = g.executeGremlinScript(qry.queryStr, qry.isPathExpression); + if(debug) { + println(" rawRes " +rawRes) + } if (!qry.hasSelectList) { val rows = rawRes.asInstanceOf[java.util.List[AnyRef]].map { v => - val iV = instanceObject(v) - val o = persistenceStrategy.constructInstance(oType, iV) - addPathStruct(v, o) + val instObj = instanceObject(v) + val o = persistenceStrategy.constructInstance(oType, instObj) + addPathStruct(v, o) } GremlinQueryResult(qry.expr.toString, rType, rows.toList) } else { val sType = oType.asInstanceOf[StructType] val rows = rawRes.asInstanceOf[java.util.List[AnyRef]].map { r => - val rV = instanceObject(r).asInstanceOf[Row[java.util.List[AnyRef]]] + val rV = instanceObject(r) val sInstance = sType.createInstance() val selObj = SelectExpressionHelper.extractSelectExpression(qry.expr) - if (selObj.isDefined) - { + if (selObj.isDefined) { val selExpr = selObj.get.asInstanceOf[Expressions.SelectExpression] - selExpr.selectListWithAlias.foreach { aE => + selExpr.selectListWithAlias.foreach { aE => val cName = aE.alias val (src, idx) = qry.resultMaping(cName) - val v = rV.getColumn(src).get(idx) + val v = getColumnValue(rV, src, idx) sInstance.set(cName, persistenceStrategy.constructInstance(aE.dataType, v)) } - } - addPathStruct(r, sInstance) + } + addPathStruct(r, sInstance) } GremlinQueryResult(qry.expr.toString, rType, rows.toList) } } + + private def getColumnValue(rowValue: AnyRef, colName: String, idx: Integer) : AnyRef = { + + var rawColumnValue: AnyRef = null; + if(rowValue.isInstanceOf[java.util.Map[_,_]]) { + val columnsMap = rowValue.asInstanceOf[java.util.Map[String,AnyRef]]; + rawColumnValue = columnsMap.get(colName); + } + else { + //when there is only one column, result does not come back as a map + rawColumnValue = rowValue; + } + + var value : AnyRef = null; + if(rawColumnValue.isInstanceOf[java.util.List[_]] && idx >= 0) { + val arr = rawColumnValue.asInstanceOf[java.util.List[AnyRef]]; + value = arr.get(idx); + } + else { + value = rawColumnValue; + } + + return value; + } } object JsonHelper { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d2d6ff7d/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala ---------------------------------------------------------------------- diff --git a/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala b/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala index d336f1e..447622b 100755 --- a/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala +++ b/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala @@ -18,16 +18,30 @@ package org.apache.atlas.query -import org.apache.atlas.query.TypeUtils.FieldInfo -import org.apache.atlas.query.Expressions._ -import org.apache.atlas.typesystem.types.{DataTypes, TypeSystem} -import org.apache.atlas.typesystem.types.DataTypes.TypeCategory -import org.joda.time.format.ISODateTimeFormat - import scala.collection.mutable import scala.collection.mutable.ArrayBuffer + +import org.apache.atlas.query.TypeUtils.FieldInfo; +import org.apache.atlas.query.Expressions._ +import org.apache.atlas.repository.graphdb.GremlinVersion +import org.apache.atlas.typesystem.types.DataTypes +import org.apache.atlas.typesystem.types.DataTypes.TypeCategory import org.apache.atlas.typesystem.types.IDataType import org.apache.commons.lang.StringEscapeUtils +import org.apache.atlas.typesystem.types.TypeSystem +import org.apache.atlas.typesystem.types.AttributeInfo +import org.joda.time.format.ISODateTimeFormat +import org.apache.atlas.typesystem.types.DataTypes.BigDecimalType +import org.apache.atlas.typesystem.types.DataTypes.ByteType +import org.apache.atlas.typesystem.types.DataTypes.BooleanType +import org.apache.atlas.typesystem.types.DataTypes.DateType +import org.apache.atlas.typesystem.types.DataTypes.BigIntegerType +import org.apache.atlas.typesystem.types.DataTypes.IntType +import org.apache.atlas.typesystem.types.DataTypes.StringType +import org.apache.atlas.typesystem.types.DataTypes.LongType +import org.apache.atlas.typesystem.types.DataTypes.DoubleType +import org.apache.atlas.typesystem.types.DataTypes.FloatType +import org.apache.atlas.typesystem.types.DataTypes.ShortType trait IntSequence { def next: Int @@ -37,10 +51,11 @@ case class GremlinQuery(expr: Expression, queryStr: String, resultMaping: Map[St def hasSelectList = resultMaping != null - def isPathExpresion = expr.isInstanceOf[PathExpression] - + def isPathExpression = expr.isInstanceOf[PathExpression] + } + trait SelectExpressionHandling { /** @@ -112,8 +127,8 @@ trait SelectExpressionHandling { /** * For each Output Column in the SelectExpression compute the ArrayList(Src) this maps to and the position within * this list. - * - * @param sel + * + * @param sel * @return */ def buildResultMapping(sel: SelectExpression): Map[String, (String, Int)] = { @@ -207,14 +222,14 @@ class GremlinTranslator(expr: Expression, stats.last } - def escape(str: String): String = { - if (str != null) { - return str.replace("\"", "\\\"").replace("$", "\\$"); + def escape(str: String): String = { + if (str != null) { + return str.replace("\"", "\\\"").replace("$", "\\$"); + } + str } - str - } - - private def genQuery(expr: Expression, inSelect: Boolean): String = expr match { + + private def genQuery(expr: Expression, inSelect: Boolean): String = expr match { case ClassExpression(clsName) => typeTestExpression(clsName) case TraitExpression(clsName) => @@ -222,71 +237,114 @@ class GremlinTranslator(expr: Expression, case fe@FieldExpression(fieldName, fInfo, child) if fe.dataType.getTypeCategory == TypeCategory.PRIMITIVE || fe.dataType.getTypeCategory == TypeCategory.ARRAY => { val fN = "\"" + gPersistenceBehavior.fieldNameInVertex(fInfo.dataType, fInfo.attrInfo) + "\"" - child match { - case Some(e) => s"${genQuery(e, inSelect)}.$fN" - case None => s"$fN" - } + genPropertyAccessExpr(child, fInfo, fN, inSelect) + } case fe@FieldExpression(fieldName, fInfo, child) if fe.dataType.getTypeCategory == TypeCategory.CLASS || fe.dataType.getTypeCategory == TypeCategory.STRUCT => { val direction = if (fInfo.isReverse) "in" else "out" val edgeLbl = gPersistenceBehavior.edgeLabel(fInfo) val step = s"""$direction("$edgeLbl")""" - child match { - case Some(e) => s"${genQuery(e, inSelect)}.$step" - case None => step - } + generateAndPrependExpr(child, inSelect, s"""$step""") } case fe@FieldExpression(fieldName, fInfo, child) if fInfo.traitName != null => { val direction = gPersistenceBehavior.instanceToTraitEdgeDirection val edgeLbl = gPersistenceBehavior.edgeLabel(fInfo) val step = s"""$direction("$edgeLbl")""" - child match { - case Some(e) => s"${genQuery(e, inSelect)}.$step" - case None => step - } + generateAndPrependExpr(child, inSelect, s"""$step""") } case c@ComparisonExpression(symb, f@FieldExpression(fieldName, fInfo, ch), l) => { - val qualifiedPropertyName = s"${gPersistenceBehavior.fieldNameInVertex(fInfo.dataType, fInfo.attrInfo)}" - val persistentExprValue = translateValueToPersistentForm(fInfo, l) - return generateAndPrependExpr(ch, inSelect, s"""has("${qualifiedPropertyName}", ${gPersistenceBehavior.gremlinCompOp(c)}, $persistentExprValue)""") + return genHasPredicate(ch, fInfo, fieldName, inSelect, c, l) } case fil@FilterExpression(child, condExpr) => { s"${genQuery(child, inSelect)}.${genQuery(condExpr, inSelect)}" } case l@LogicalExpression(symb, children) => { - s"""$symb${children.map("_()." + genQuery(_, inSelect)).mkString("(", ",", ")")}""" + if(gPersistenceBehavior.getSupportedGremlinVersion() == GremlinVersion.THREE) { + if(children.length == 1) { + //gremlin 3 treats one element expressions as 'false'. Avoid + //creating a boolean expression in this case. Inline the expression + //note: we can't simply omit it, since it will cause us to traverse the edge! + //use 'where' instead + var child : Expression = children.head; + //if child is a back expression, that expression becomes an argument to where + + return s"""where(${genQuery(child, inSelect)})"""; + } + else { + // Gremlin 3 does not support _() syntax + // + return s"""$symb${children.map( genQuery(_, inSelect)).mkString("(", ",", ")")}""" + } + } + else { + s"""$symb${children.map("_()." + genQuery(_, inSelect)).mkString("(", ",", ")")}""" + } } case sel@SelectExpression(child, selList) => { - val m = groupSelectExpressionsBySrc(sel) - var srcNamesList: List[String] = List() - var srcExprsList: List[List[String]] = List() - val it = m.iterator - while (it.hasNext) { - val (src, selExprs) = it.next - srcNamesList = srcNamesList :+ s""""$src"""" - srcExprsList = srcExprsList :+ selExprs.map { selExpr => - genQuery(selExpr, true) - } + val m = groupSelectExpressionsBySrc(sel) + var srcNamesList: List[String] = List() + var srcExprsList: List[List[String]] = List() + val it = m.iterator + while (it.hasNext) { + val (src, selExprs) = it.next + srcNamesList = srcNamesList :+ s""""$src"""" + srcExprsList = srcExprsList :+ selExprs.map { selExpr => + genQuery(selExpr, true) + } + } + val srcExprsStringList = srcExprsList.map { + _.mkString("[", ",", "]") + } + + if(gPersistenceBehavior.getSupportedGremlinVersion() == GremlinVersion.TWO) { + val srcNamesString = srcNamesList.mkString("[", ",", "]") + val srcExprsString = srcExprsStringList.foldLeft("")(_ + "{" + _ + "}") + s"${genQuery(child, inSelect)}.select($srcNamesString)$srcExprsString" + } + else { + //gremlin 3 + val srcNamesString = srcNamesList.mkString("", ",", "") + val srcExprsString = srcExprsStringList.foldLeft("")(_ + ".by({" + _ + "} as Function)") + s"${genQuery(child, inSelect)}.select($srcNamesString)$srcExprsString" + } + } + case loop@LoopExpression(input, loopExpr, t) => { + + if(gPersistenceBehavior.getSupportedGremlinVersion() == GremlinVersion.TWO) { + val inputQry = genQuery(input, inSelect) + val loopingPathGExpr = genQuery(loopExpr, inSelect) + val loopGExpr = s"""loop("${input.asInstanceOf[AliasExpression].alias}")""" + val untilCriteria = if (t.isDefined) s"{it.loops < ${t.get.value}}" else "{it.path.contains(it.object)?false:true}" + val loopObjectGExpr = gPersistenceBehavior.loopObjectExpression(input.dataType) + val enablePathExpr = s".enablePath()" + s"""${inputQry}.${loopingPathGExpr}.${loopGExpr}${untilCriteria}${loopObjectGExpr}${enablePathExpr}""" } - val srcNamesString = srcNamesList.mkString("[", ",", "]") - val srcExprsStringList = srcExprsList.map { - _.mkString("[", ",", "]") + else { + //gremlin 3 - TODO - add support for circular lineage + val inputQry = genQuery(input, inSelect) + val repeatExpr = s"""repeat(__.${genQuery(loopExpr, inSelect)})""" + val optTimesExpr = if (t.isDefined) s".times(${t.get.value})" else "" + val emitExpr = s""".emit(${gPersistenceBehavior.loopObjectExpression(input.dataType)})""" + + s"""${inputQry}.${repeatExpr}${optTimesExpr}${emitExpr}""" + + } + } + case BackReference(alias, _, _) => { + + if (inSelect) { + gPersistenceBehavior.fieldPrefixInSelect() + } + else { + if(gPersistenceBehavior.getSupportedGremlinVersion() == GremlinVersion.TWO) { + s"""back("$alias")""" + } + else { + s"""select("$alias")""" + } } - val srcExprsString = srcExprsStringList.foldLeft("")(_ + "{" + _ + "}") - s"${genQuery(child, inSelect)}.select($srcNamesString)$srcExprsString" } - case loop@LoopExpression(input, loopExpr, t) => { - val inputQry = genQuery(input, inSelect) - val loopingPathGExpr = genQuery(loopExpr, inSelect) - val loopGExpr = s"""loop("${input.asInstanceOf[AliasExpression].alias}")""" - val untilCriteria = if (t.isDefined) s"{it.loops < ${t.get.value}}" else "{it.path.contains(it.object)?false:true}" - val loopObjectGExpr = gPersistenceBehavior.loopObjectExpression(input.dataType) - val enablePathExpr = s".enablePath()" - s"""${inputQry}.${loopingPathGExpr}.${loopGExpr}${untilCriteria}${loopObjectGExpr}${enablePathExpr}""" - } - case BackReference(alias, _, _) => - if (inSelect) gPersistenceBehavior.fieldPrefixInSelect else s"""back("$alias")""" case AliasExpression(child, alias) => s"""${genQuery(child, inSelect)}.as("$alias")""" case isTraitLeafExpression(traitName, Some(clsExp)) => s"""out("${gPersistenceBehavior.traitLabel(clsExp.dataType, traitName)}")""" @@ -294,12 +352,15 @@ class GremlinTranslator(expr: Expression, s"""out("${gPersistenceBehavior.traitLabel(child.dataType, traitName)}")""" case hasFieldLeafExpression(fieldName, clsExp) => clsExp match { case None => s"""has("$fieldName")""" - case Some(x) => - x match { - case c: ClassExpression => - s"""has("${x.asInstanceOf[ClassExpression].clsName}.$fieldName")""" - case default => s"""has("$fieldName")""" - } + case Some(x) => { + val fi = TypeUtils.resolveReference(clsExp.get.dataType, fieldName); + if(! fi.isDefined) { + s"""has("$fieldName")""" + } + else { + s"""has("${gPersistenceBehavior.fieldNameInVertex(fi.get.dataType, fi.get.attrInfo)}")""" + } + } } case hasFieldUnaryExpression(fieldName, child) => s"""${genQuery(child, inSelect)}.has("$fieldName")""" @@ -314,73 +375,92 @@ class GremlinTranslator(expr: Expression, s"${genQuery(child, inSelect)}" } case pe@PathExpression(child) => { - s"${genQuery(child, inSelect)}.path" + if(gPersistenceBehavior.getSupportedGremlinVersion() == GremlinVersion.TWO) { + s"${genQuery(child, inSelect)}.path" + } + else { + s"${genQuery(child, inSelect)}.path()" + } } case order@OrderExpression(child, odr, asc) => { + var orderby = "" var orderExpression = odr - if(odr.isInstanceOf[BackReference]) { orderExpression = odr.asInstanceOf[BackReference].reference } - else if (odr.isInstanceOf[AliasExpression]) { orderExpression = odr.asInstanceOf[AliasExpression].child} + if(odr.isInstanceOf[BackReference]) { + orderExpression = odr.asInstanceOf[BackReference].reference + } + else if (odr.isInstanceOf[AliasExpression]) { + orderExpression = odr.asInstanceOf[AliasExpression].child + } val orderbyProperty = genQuery(orderExpression, false) - val bProperty = s"it.b.$orderbyProperty" - val aProperty = s"it.a.$orderbyProperty" - val aCondition = s"($aProperty != null ? $aProperty.toLowerCase(): $aProperty)" - val bCondition = s"($bProperty != null ? $bProperty.toLowerCase(): $bProperty)" - var orderby = "" - asc match { - //builds a closure comparison function based on provided order by clause in DSL. This will be used to sort the results by gremlin order pipe. - //Ordering is case insensitive. - case false=> orderby = s"order{$bCondition <=> $aCondition}"//descending - case _ => orderby = s"order{$aCondition <=> $bCondition}" - } + if(gPersistenceBehavior.getSupportedGremlinVersion() == GremlinVersion.TWO) { + + val bProperty = s"it.b.$orderbyProperty" + val aProperty = s"it.a.$orderbyProperty" + val aCondition = s"($aProperty != null ? $aProperty.toLowerCase(): $aProperty)" + val bCondition = s"($bProperty != null ? $bProperty.toLowerCase(): $bProperty)" + orderby = asc match { + //builds a closure comparison function based on provided order by clause in DSL. This will be used to sort the results by gremlin order pipe. + //Ordering is case insensitive. + case false=> s"order{$bCondition <=> $aCondition}"//descending + case _ => s"order{$aCondition <=> $bCondition}" + } + } + else { + val orderbyProperty = genQuery(orderExpression, true); + val aPropertyExpr = gremlin3ToLowerCase("a"); + val bPropertyExpr = gremlin3ToLowerCase("b"); + + orderby = asc match { + //builds a closure comparison function based on provided order by clause in DSL. This will be used to sort the results by gremlin order pipe. + //Ordering is case insensitive. + case false=> s"""order().by({$orderbyProperty'}, { a,b -> $bPropertyExpr <=> $aPropertyExpr })""" + case _ => s"""order().by({$orderbyProperty},{ a,b -> $aPropertyExpr <=> $bPropertyExpr })""" + + } + } s"""${genQuery(child, inSelect)}.$orderby""" } case limitOffset@LimitExpression(child, limit, offset) => { - val totalResultRows = limit.value + offset.value - s"""${genQuery(child, inSelect)} [$offset..<$totalResultRows]""" + if(gPersistenceBehavior.getSupportedGremlinVersion() == GremlinVersion.TWO) { + val totalResultRows = limit.value + offset.value + s"""${genQuery(child, inSelect)} [$offset..<$totalResultRows]""" + } + else { + val totalResultRows = limit.value + offset.value + s"""${genQuery(child, inSelect)}.range($offset,$totalResultRows)""" + } } case x => throw new GremlinTranslationException(x, "expression not yet supported") } + def gremlin3ToLowerCase(varName : String) : String = { + s"""($varName != null ? $varName.toString().toLowerCase() : null)""" + } + + def genPropertyAccessExpr(e: Option[Expression], fInfo : FieldInfo, quotedPropertyName: String, inSelect: Boolean) : String = { - def translateValueToPersistentForm(fInfo: FieldInfo, l: Expression): Any = { - - val dataType = fInfo.attrInfo.dataType; - val QUOTE = "\""; - - if (dataType == DataTypes.DATE_TYPE) { - try { - //Accepts both date, datetime formats - val dateStr = l.toString.stripPrefix(QUOTE).stripSuffix(QUOTE) - val dateVal = ISODateTimeFormat.dateOptionalTimeParser().parseDateTime(dateStr).getMillis - return dateVal - } catch { - case pe: java.text.ParseException => - throw new GremlinTranslationException(l, - "Date format " + l + " not supported. Should be of the format " + - TypeSystem.getInstance().getDateFormat.toPattern); - } - } - else if(dataType == DataTypes.BYTE_TYPE) { - //cast needed, otherwise get class cast exception when trying to compare, since the - //persist value is assumed to be an Integer - return s"""(byte)$l""" - } - else if(dataType == DataTypes.SHORT_TYPE) { - return s"""(short)$l""" - } - else if(dataType == DataTypes.LONG_TYPE) { - return s"""${l}L""" - } - else if(dataType == DataTypes.FLOAT_TYPE) { - return s"""${l}f""" - } - else if(dataType == DataTypes.DOUBLE_TYPE) { - return s"""${l}d""" - } else if(dataType == DataTypes.STRING_TYPE) { - return string(escape(l.toString.stripPrefix(QUOTE).stripSuffix(QUOTE))); - } else { - l - } + if(gPersistenceBehavior.getSupportedGremlinVersion() == GremlinVersion.TWO) { + generateAndPrependExpr(e, inSelect, s"""$quotedPropertyName""") + } + else { + val attrInfo : AttributeInfo = fInfo.attrInfo; + val attrType : IDataType[_] = attrInfo.dataType; + if(inSelect) { + val expr = generateAndPrependExpr(e, inSelect, s"""property($quotedPropertyName).orElse(null)"""); + return gPersistenceBehavior.generatePersisentToLogicalConversionExpression(expr, attrType); + } + else { + val unmapped = s"""values($quotedPropertyName)""" + val expr = if(gPersistenceBehavior.isPropertyValueConversionNeeded(attrType)) { + val conversionFunction = gPersistenceBehavior.generatePersisentToLogicalConversionExpression(s"""it.get()""", attrType); + s"""$unmapped.map{ $conversionFunction }""" + } + else { + unmapped + } + generateAndPrependExpr(e, inSelect, expr) + } + } } def generateAndPrependExpr(e1: Option[Expression], inSelect: Boolean, e2: String) : String = e1 match { @@ -389,49 +469,120 @@ class GremlinTranslator(expr: Expression, case None => e2 } - def genFullQuery(expr: Expression): String = { + def genHasPredicate(e: Option[Expression], fInfo : FieldInfo, fieldName: String, inSelect: Boolean, c: ComparisonExpression, expr: Expression) : String = { + + val qualifiedPropertyName = s"${gPersistenceBehavior.fieldNameInVertex(fInfo.dataType, fInfo.attrInfo)}" + val persistentExprValue = translateValueToPersistentForm(fInfo, expr); + if(gPersistenceBehavior.getSupportedGremlinVersion() == GremlinVersion.TWO) { + return generateAndPrependExpr(e, inSelect, s"""has("${qualifiedPropertyName}", ${gPersistenceBehavior.gremlinCompOp(c)}, $persistentExprValue)"""); + } + else { + val attrInfo : AttributeInfo = fInfo.attrInfo; + val attrType : IDataType[_] = attrInfo.dataType; + if(gPersistenceBehavior.isPropertyValueConversionNeeded(attrType)) { + //for some types, the logical value cannot be stored directly in the underlying graph, + //and conversion logic is needed to convert the persistent form of the value + //to the actual value. In cases like this, we generate a conversion expression to + //do this conversion and use the filter step to perform the comparsion in the gremlin query + val vertexExpr = "((Vertex)it.get())"; + val conversionExpr = gPersistenceBehavior.generatePersisentToLogicalConversionExpression(s"""$vertexExpr.value("$qualifiedPropertyName")""", attrType); + return generateAndPrependExpr(e, inSelect, s"""filter{$vertexExpr.property("$qualifiedPropertyName").isPresent() && $conversionExpr ${gPersistenceBehavior.gremlinPrimitiveOp(c)} $persistentExprValue}"""); + } + else { + return generateAndPrependExpr(e, inSelect, s"""has("${qualifiedPropertyName}", ${gPersistenceBehavior.gremlinCompOp(c)}($persistentExprValue))"""); + } + } + + } + + def translateValueToPersistentForm(fInfo: FieldInfo, l: Expression): Any = { + + val dataType = fInfo.attrInfo.dataType; + val QUOTE = "\""; + + if (dataType == DataTypes.DATE_TYPE) { + try { + //Accepts both date, datetime formats + val dateStr = l.toString.stripPrefix(QUOTE).stripSuffix(QUOTE) + val dateVal = ISODateTimeFormat.dateOptionalTimeParser().parseDateTime(dateStr).getMillis + return dateVal + } catch { + case pe: java.text.ParseException => + throw new GremlinTranslationException(l, + "Date format " + l + " not supported. Should be of the format " + + TypeSystem.getInstance().getDateFormat.toPattern); + } + } + else if(dataType == DataTypes.BYTE_TYPE) { + //cast needed, otherwise get class cast exception when trying to compare, since the + //persist value is assumed to be an Integer + return s"""(byte)$l""" + } + else if(dataType == DataTypes.SHORT_TYPE) { + return s"""(short)$l""" + } + else if(dataType == DataTypes.LONG_TYPE) { + return s"""${l}L""" + } + else if(dataType == DataTypes.FLOAT_TYPE) { + return s"""${l}f""" + } + else if(dataType == DataTypes.DOUBLE_TYPE) { + return s"""${l}d""" + } + else if(dataType == DataTypes.STRING_TYPE) { + return string(escape(l.toString.stripPrefix(QUOTE).stripSuffix(QUOTE))); + } + else { + return l + } + } + + def genFullQuery(expr: Expression, hasSelect: Boolean): String = { + var q = genQuery(expr, false) + val debug:Boolean = false if(gPersistenceBehavior.addGraphVertexPrefix(preStatements)) { - q = s"g.V.$q" + q = s"g.V()${gPersistenceBehavior.initialQueryCondition}.$q" } - q = s"$q.toList()" - + q = s"$q.toList()${gPersistenceBehavior.getGraph().getOutputTransformationPredicate(hasSelect, expr.isInstanceOf[PathExpression])}" + + if(debug) { + println(" query " + q) + } + q = (preStatements ++ Seq(q) ++ postStatements).mkString("", ";", "") + /* * the L:{} represents a groovy code block; the label is needed * to distinguish it from a groovy closure. */ s"L:{$q}" + } - def translate(): GremlinQuery = { var e1 = expr.transformUp(wrapAndRule) e1.traverseUp(validateComparisonForm) - e1 = e1.transformUp(new AddAliasToSelectInput) e1.traverseUp(validateSelectExprHaveOneSrc) e1 = e1.transformUp(addAliasToLoopInput()) e1 = e1.transformUp(instanceClauseToTop(e1)) e1 = e1.transformUp(traitClauseWithInstanceForTop(e1)) - + //Following code extracts the select expressions from expression tree. - - val se = SelectExpressionHelper.extractSelectExpression(e1) - if (se.isDefined) - { - val rMap = buildResultMapping(se.get) - GremlinQuery(e1, genFullQuery(e1), rMap) - } - else - { - GremlinQuery(e1, genFullQuery(e1), null) - } + + val se = SelectExpressionHelper.extractSelectExpression(e1) + if (se.isDefined) { + val rMap = buildResultMapping(se.get) + GremlinQuery(e1, genFullQuery(e1, true), rMap) + } else { + GremlinQuery(e1, genFullQuery(e1, false), null) } } - +} object SelectExpressionHelper { /** * This method extracts the child select expression from parent expression @@ -453,7 +604,7 @@ class GremlinTranslator(expr: Expression, case _ => { None } - + } } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d2d6ff7d/repository/src/main/scala/org/apache/atlas/query/QueryProcessor.scala ---------------------------------------------------------------------- diff --git a/repository/src/main/scala/org/apache/atlas/query/QueryProcessor.scala b/repository/src/main/scala/org/apache/atlas/query/QueryProcessor.scala index 0d2a908..ee95a20 100755 --- a/repository/src/main/scala/org/apache/atlas/query/QueryProcessor.scala +++ b/repository/src/main/scala/org/apache/atlas/query/QueryProcessor.scala @@ -18,21 +18,27 @@ package org.apache.atlas.query -import com.thinkaurelius.titan.core.TitanGraph +import org.apache.atlas.repository.graphdb.AtlasGraph import org.apache.atlas.query.Expressions._ import org.slf4j.{Logger, LoggerFactory} object QueryProcessor { val LOG : Logger = LoggerFactory.getLogger("org.apache.atlas.query.QueryProcessor") - def evaluate(e: Expression, g: TitanGraph, gP : GraphPersistenceStrategies = GraphPersistenceStrategy1): + def evaluate(e: Expression, g: AtlasGraph[_,_], gP : GraphPersistenceStrategies = null): GremlinQueryResult = { + + var strategy = gP; + if(strategy == null) { + strategy = GraphPersistenceStrategy1(g); + } + val e1 = validate(e) - val q = new GremlinTranslator(e1, gP).translate() + val q = new GremlinTranslator(e1, strategy).translate() LOG.debug("Query: " + e1) LOG.debug("Expression Tree:\n" + e1.treeString) LOG.debug("Gremlin Query: " + q.queryStr) - new GremlinEvaluator(q, gP, g).evaluate() + new GremlinEvaluator(q, strategy, g).evaluate() } def validate(e: Expression): Expression = { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d2d6ff7d/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java b/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java index 500a305..d7068cd 100644 --- a/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java +++ b/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java @@ -17,15 +17,15 @@ */ package org.apache.atlas; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.thinkaurelius.titan.core.TitanGraph; -import com.thinkaurelius.titan.core.util.TitanCleanup; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import javax.inject.Inject; import org.apache.atlas.repository.MetadataRepository; +import org.apache.atlas.repository.graph.AtlasGraphProvider; import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; -import org.apache.atlas.repository.graph.GraphProvider; import org.apache.atlas.services.MetadataService; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.Referenceable; @@ -45,11 +45,9 @@ import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.testng.annotations.Guice; -import javax.inject.Inject; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; /** * Base Class to set up hive types and instances for tests @@ -63,29 +61,21 @@ public class BaseRepositoryTest { @Inject protected MetadataRepository repository; - @Inject - protected GraphProvider<TitanGraph> graphProvider; + protected void setUp() throws Exception { + //force graph initialization / built in type registration + TestUtils.getGraph(); setUpTypes(); - new GraphBackedSearchIndexer(graphProvider); - RequestContext.createContext(); + new GraphBackedSearchIndexer(); + TestUtils.resetRequestContext(); setupInstances(); - TestUtils.dumpGraph(graphProvider.get()); + TestUtils.dumpGraph(TestUtils.getGraph()); } - + protected void tearDown() throws Exception { TypeSystem.getInstance().reset(); - try { - graphProvider.get().shutdown(); - } catch (Exception e) { - e.printStackTrace(); - } - try { - TitanCleanup.clear(graphProvider.get()); - } catch (Exception e) { - e.printStackTrace(); - } + AtlasGraphProvider.cleanup(); } private void setUpTypes() throws Exception { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d2d6ff7d/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java b/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java index 4195955..6e13077 100755 --- a/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java +++ b/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java @@ -18,14 +18,11 @@ package org.apache.atlas; -import com.thinkaurelius.titan.core.TitanGraph; -import org.apache.atlas.repository.graph.GraphProvider; +import org.apache.atlas.repository.graph.AtlasGraphProvider; import org.testng.Assert; import org.testng.annotations.Guice; import org.testng.annotations.Test; -import javax.inject.Inject; - /** * Unit test for Guice injector service loading * @@ -35,12 +32,8 @@ import javax.inject.Inject; @Guice(modules = RepositoryMetadataModule.class) public class RepositoryServiceLoadingTest { - @Inject - private GraphProvider<TitanGraph> graphProvider; - @Test public void testGetGraphService() throws Exception { - Assert.assertNotNull(graphProvider); - Assert.assertNotNull(graphProvider.get()); + Assert.assertNotNull(AtlasGraphProvider.getGraphInstance()); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d2d6ff7d/repository/src/test/java/org/apache/atlas/TestUtils.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/TestUtils.java b/repository/src/test/java/org/apache/atlas/TestUtils.java index b27854e..30071ba 100755 --- a/repository/src/test/java/org/apache/atlas/TestUtils.java +++ b/repository/src/test/java/org/apache/atlas/TestUtils.java @@ -18,12 +18,36 @@ package org.apache.atlas; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.thinkaurelius.titan.core.TitanGraph; -import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter; +import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef; +import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef; +import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef; +import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructTypeDef; +import static org.apache.atlas.typesystem.types.utils.TypesUtil.createTraitTypeDef; +import static org.apache.atlas.typesystem.types.utils.TypesUtil.createUniqueRequiredAttrDef; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +import org.apache.atlas.listener.EntityChangeListener; +import org.apache.atlas.listener.TypesChangeListener; +import org.apache.atlas.repository.MetadataRepository; +import org.apache.atlas.repository.graph.AtlasGraphProvider; +import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; import org.apache.atlas.repository.graph.GraphHelper; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.typestore.GraphBackedTypeStore; +import org.apache.atlas.repository.typestore.ITypeStore; +import org.apache.atlas.services.DefaultMetadataService; import org.apache.atlas.services.MetadataService; +import org.apache.atlas.services.ReservedTypesRegistrar; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.TypesDef; @@ -40,25 +64,18 @@ import org.apache.atlas.typesystem.types.Multiplicity; import org.apache.atlas.typesystem.types.StructTypeDefinition; import org.apache.atlas.typesystem.types.TraitType; import org.apache.atlas.typesystem.types.TypeSystem; +import org.apache.atlas.typesystem.types.cache.DefaultTypeCache; +import org.apache.atlas.typesystem.types.cache.TypeCache; import org.apache.atlas.typesystem.types.utils.TypesUtil; +import org.apache.atlas.util.AtlasRepositoryConfiguration; +import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.RandomStringUtils; import org.codehaus.jettison.json.JSONArray; import org.testng.Assert; -import java.io.File; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Date; -import java.util.List; - -import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef; -import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef; -import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef; -import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructTypeDef; -import static org.apache.atlas.typesystem.types.utils.TypesUtil.createTraitTypeDef; -import static org.apache.atlas.typesystem.types.utils.TypesUtil.createUniqueRequiredAttrDef; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.inject.Provider; /** * Test utility class. @@ -73,16 +90,30 @@ public final class TestUtils { /** * Dumps the graph in GSON format in the path returned. * - * @param titanGraph handle to graph + * @param graph handle to graph * @return path to the dump file * @throws Exception */ - public static String dumpGraph(TitanGraph titanGraph) throws Exception { + public static String dumpGraph(AtlasGraph<?,?> graph) throws Exception { File tempFile = File.createTempFile("graph", ".gson"); System.out.println("tempFile.getPath() = " + tempFile.getPath()); - GraphSONWriter.outputGraph(titanGraph, tempFile.getPath()); + GraphHelper.dumpToLog(graph); + FileOutputStream os = null; + try { + os = new FileOutputStream(tempFile); + graph.exportToGson(os); + } + finally { + if(os != null) { + try { + os.close(); + } + catch(IOException e) { + e.printStackTrace(); + } + } + } - GraphHelper.dumpToLog(titanGraph); return tempFile.getPath(); } @@ -469,4 +500,44 @@ public final class TestUtils { } return null; } + + public static void resetRequestContext() { + RequestContext.createContext(); + } + + public static void setupGraphProvider(MetadataRepository repo) throws AtlasException { + TypeCache typeCache = null; + try { + typeCache = AtlasRepositoryConfiguration.getTypeCache().newInstance(); + } + catch(Throwable t) { + typeCache = new DefaultTypeCache(); + } + final GraphBackedSearchIndexer indexer = new GraphBackedSearchIndexer(); + Provider<TypesChangeListener> indexerProvider = new Provider<TypesChangeListener>() { + + @Override + public TypesChangeListener get() { + return indexer; + } + }; + + Configuration config = ApplicationProperties.get(); + ITypeStore typeStore = new GraphBackedTypeStore(); + DefaultMetadataService defaultMetadataService = new DefaultMetadataService(repo, + typeStore, + new ReservedTypesRegistrar(), + Collections.singletonList(indexerProvider), + new ArrayList<Provider<EntityChangeListener>>(), TypeSystem.getInstance(), config, typeCache); + + //commit the created types + getGraph().commit(); + + } + + public static AtlasGraph getGraph() { + + return AtlasGraphProvider.getGraphInstance(); + + } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d2d6ff7d/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java index 40dc861..13b7d22 100755 --- a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java @@ -18,7 +18,23 @@ package org.apache.atlas.discovery; -import com.google.common.collect.ImmutableSet; +import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef; +import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef; +import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.inject.Inject; + import org.apache.atlas.AtlasException; import org.apache.atlas.BaseRepositoryTest; import org.apache.atlas.RepositoryMetadataModule; @@ -28,8 +44,8 @@ import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; import org.apache.atlas.query.QueryParams; import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.MetadataRepository; +import org.apache.atlas.repository.graph.AtlasGraphProvider; import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; -import org.apache.atlas.repository.graph.TitanGraphProvider; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.persistence.Id; @@ -49,21 +65,7 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Guice; import org.testng.annotations.Test; -import javax.inject.Inject; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef; -import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef; -import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotNull; +import com.google.common.collect.ImmutableSet; @Guice(modules = RepositoryMetadataModule.class) public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { @@ -75,6 +77,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { private GraphBackedDiscoveryService discoveryService; private QueryParams queryParams = new QueryParams(40, 0); + @Override @BeforeClass public void setUp() throws Exception { super.setUp(); @@ -112,11 +115,11 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { } } - TitanGraphProvider provider = new TitanGraphProvider(); + //We need to commit the transaction before creating the indices to release the locks held by the transaction. //otherwise, the index commit will fail while waiting for the those locks to be released. - provider.get().commit(); - GraphBackedSearchIndexer idx = new GraphBackedSearchIndexer(provider); + AtlasGraphProvider.getGraphInstance().commit(); + GraphBackedSearchIndexer idx = new GraphBackedSearchIndexer(); idx.onAdd(newTypes); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d2d6ff7d/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java index 550a98e..6de995b 100644 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java @@ -18,20 +18,43 @@ package org.apache.atlas.repository.graph; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.thinkaurelius.titan.core.TitanGraph; -import com.thinkaurelius.titan.core.util.TitanCleanup; -import com.tinkerpop.blueprints.Vertex; +import static org.apache.atlas.TestUtils.COLUMNS_ATTR_NAME; +import static org.apache.atlas.TestUtils.COLUMN_TYPE; +import static org.apache.atlas.TestUtils.NAME; +import static org.apache.atlas.TestUtils.PII; +import static org.apache.atlas.TestUtils.PROCESS_TYPE; +import static org.apache.atlas.TestUtils.TABLE_TYPE; +import static org.apache.atlas.TestUtils.createColumnEntity; +import static org.apache.atlas.TestUtils.createDBEntity; +import static org.apache.atlas.TestUtils.createTableEntity; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasClient.EntityResult; import org.apache.atlas.AtlasException; +import org.apache.atlas.GraphTransaction; import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.RequestContext; import org.apache.atlas.TestUtils; import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.MetadataRepository; import org.apache.atlas.repository.RepositoryException; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.atlas.typesystem.IStruct; import org.apache.atlas.typesystem.ITypedReferenceableInstance; @@ -61,28 +84,8 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Guice; import org.testng.annotations.Test; -import javax.inject.Inject; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.apache.atlas.TestUtils.COLUMNS_ATTR_NAME; -import static org.apache.atlas.TestUtils.COLUMN_TYPE; -import static org.apache.atlas.TestUtils.NAME; -import static org.apache.atlas.TestUtils.PII; -import static org.apache.atlas.TestUtils.PROCESS_TYPE; -import static org.apache.atlas.TestUtils.TABLE_TYPE; -import static org.apache.atlas.TestUtils.createColumnEntity; -import static org.apache.atlas.TestUtils.createDBEntity; -import static org.apache.atlas.TestUtils.createTableEntity; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotEquals; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; /** * Test for GraphBackedMetadataRepository.deleteEntities @@ -93,10 +96,7 @@ import static org.testng.Assert.fail; @Guice(modules = RepositoryMetadataModule.class) public abstract class GraphBackedMetadataRepositoryDeleteTestBase { - @Inject - private GraphProvider<TitanGraph> graphProvider; - - protected GraphBackedMetadataRepository repositoryService; + protected MetadataRepository repositoryService; private TypeSystem typeSystem; @@ -106,12 +106,50 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { @BeforeClass public void setUp() throws Exception { + typeSystem = TypeSystem.getInstance(); typeSystem.reset(); - new GraphBackedSearchIndexer(graphProvider); + new GraphBackedSearchIndexer(); + final GraphBackedMetadataRepository delegate = new GraphBackedMetadataRepository(getDeleteHandler(typeSystem)); + + repositoryService = (MetadataRepository)Proxy.newProxyInstance(Thread.currentThread().getContextClassLoader(), + new Class[]{MetadataRepository.class}, new InvocationHandler() { + + @Override + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + boolean useTransaction = GraphBackedMetadataRepository.class.getMethod( + method.getName(), method.getParameterTypes()) + .isAnnotationPresent(GraphTransaction.class); + try { + + Object result = method.invoke(delegate, args); + if(useTransaction) { + System.out.println("Committing changes"); + TestUtils.getGraph().commit(); + System.out.println("Commit succeeded."); + } + return result; + } + catch(InvocationTargetException e) { + e.getCause().printStackTrace(); + if(useTransaction) { + System.out.println("Rolling back changes due to exception."); + TestUtils.getGraph().rollback(); + } + throw e.getCause(); + } + catch(Throwable t) { + t.printStackTrace(); + if(useTransaction) { + System.out.println("Rolling back changes due to exception."); + TestUtils.getGraph().rollback(); + } + throw t; + } + } - repositoryService = new GraphBackedMetadataRepository(graphProvider, getDeleteHandler(typeSystem)); + }); TestUtils.defineDeptEmployeeTypes(typeSystem); TestUtils.createHiveTypes(typeSystem); @@ -139,22 +177,13 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { @BeforeMethod public void setupContext() { - RequestContext.createContext(); + TestUtils.resetRequestContext(); } @AfterClass public void tearDown() throws Exception { TypeSystem.getInstance().reset(); - try { - graphProvider.get().shutdown(); - } catch (Exception e) { - e.printStackTrace(); - } - try { - TitanCleanup.clear(graphProvider.get()); - } catch (Exception e) { - e.printStackTrace(); - } + AtlasGraphProvider.cleanup(); } @Test @@ -366,7 +395,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { assertVerticesDeleted(getVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "SecurityClearance")); } - protected abstract void assertVerticesDeleted(List<Vertex> vertices); + protected abstract void assertVerticesDeleted(List<AtlasVertex> vertices); @Test public void testDeleteEntitiesWithCompositeMapReference() throws Exception { @@ -388,8 +417,8 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { String edgeLabel = GraphHelper.getEdgeLabel(compositeMapOwnerType, compositeMapOwnerType.fieldMapping.fields.get("map")); String mapEntryLabel = edgeLabel + "." + "value1"; AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(mapEntryLabel); - Vertex mapOwnerVertex = GraphHelper.getInstance().getVertexForGUID(mapOwnerGuid); - object = mapOwnerVertex.getProperty(atlasEdgeLabel.getQualifiedMapKey()); + AtlasVertex mapOwnerVertex = GraphHelper.getInstance().getVertexForGUID(mapOwnerGuid); + object = mapOwnerVertex.getProperty(atlasEdgeLabel.getQualifiedMapKey(), Object.class); Assert.assertNotNull(object); List<String> deletedEntities = deleteEntities(mapOwnerGuid).getDeletedEntities(); @@ -432,11 +461,11 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { ITypedReferenceableInstance max = repositoryService.getEntityDefinition(nameGuidMap.get("Max")); String maxGuid = max.getId()._getId(); - Vertex vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid); - Long creationTimestamp = vertex.getProperty(Constants.TIMESTAMP_PROPERTY_KEY); + AtlasVertex vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid); + Long creationTimestamp = GraphHelper.getSingleValuedProperty(vertex, Constants.TIMESTAMP_PROPERTY_KEY, Long.class); Assert.assertNotNull(creationTimestamp); - Long modificationTimestampPreUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY); + Long modificationTimestampPreUpdate = GraphHelper.getSingleValuedProperty(vertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class); Assert.assertNotNull(modificationTimestampPreUpdate); ITypedReferenceableInstance jane = repositoryService.getEntityDefinition(nameGuidMap.get("Jane")); @@ -457,7 +486,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { // Verify modification timestamp was updated. vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid); - Long modificationTimestampPostUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY); + Long modificationTimestampPostUpdate = GraphHelper.getSingleValuedProperty(vertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class); Assert.assertNotNull(modificationTimestampPostUpdate); Assert.assertTrue(creationTimestamp < modificationTimestampPostUpdate); @@ -474,7 +503,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { // Verify modification timestamp was updated. vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid); - Long modificationTimestampPost2ndUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY); + Long modificationTimestampPost2ndUpdate = GraphHelper.getSingleValuedProperty(vertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class); Assert.assertNotNull(modificationTimestampPost2ndUpdate); Assert.assertTrue(modificationTimestampPostUpdate < modificationTimestampPost2ndUpdate); @@ -764,7 +793,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { // Verify MapOwner.map attribute has expected value. String mapValueGuid = null; - Vertex mapOwnerVertex = null; + AtlasVertex mapOwnerVertex = null; mapOwnerInstance = repositoryService.getEntityDefinition(mapOwnerGuid); for (String mapAttrName : Arrays.asList("map", "biMap")) { Object object = mapOwnerInstance.get(mapAttrName); @@ -776,7 +805,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { Assert.assertNotNull(mapValueInstance); mapValueGuid = mapValueInstance.getId()._getId(); mapOwnerVertex = GraphHelper.getInstance().getVertexForGUID(mapOwnerGuid); - object = mapOwnerVertex.getProperty(atlasEdgeLabel.getQualifiedMapKey()); + object = mapOwnerVertex.getProperty(atlasEdgeLabel.getQualifiedMapKey(), Object.class); Assert.assertNotNull(object); } @@ -884,8 +913,8 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { String edgeLabel = GraphHelper.getEdgeLabel(mapOwnerType, mapOwnerType.fieldMapping.fields.get("map")); String mapEntryLabel = edgeLabel + "." + "value1"; AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(mapEntryLabel); - Vertex mapOwnerVertex = GraphHelper.getInstance().getVertexForGUID(mapOwnerGuid); - object = mapOwnerVertex.getProperty(atlasEdgeLabel.getQualifiedMapKey()); + AtlasVertex mapOwnerVertex = GraphHelper.getInstance().getVertexForGUID(mapOwnerGuid); + object = mapOwnerVertex.getProperty(atlasEdgeLabel.getQualifiedMapKey(), Object.class); Assert.assertNotNull(object); // Verify deleting the target of required map attribute throws a NullRequiredAttributeException. @@ -1076,10 +1105,11 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { repositoryService.createEntities(db, table); } - protected List<Vertex> getVertices(String propertyName, Object value) { - Iterable<Vertex> vertices = graphProvider.get().getVertices(propertyName, value); - List<Vertex> list = new ArrayList<>(); - for (Vertex vertex : vertices) { + protected List<AtlasVertex> getVertices(String propertyName, Object value) { + AtlasGraph graph = TestUtils.getGraph(); + Iterable<AtlasVertex> vertices = graph.getVertices(propertyName, value); + List<AtlasVertex> list = new ArrayList<>(); + for (AtlasVertex vertex : vertices) { list.add(vertex); } return list;
