http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala ---------------------------------------------------------------------- diff --git a/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala b/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala deleted file mode 100644 index 37015d8..0000000 --- a/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala +++ /dev/null @@ -1,806 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.query - -import java.lang.Boolean -import java.lang.Byte -import java.lang.Double -import java.lang.Float -import java.lang.Integer -import java.lang.Long -import java.lang.Short -import java.util.ArrayList - -import scala.collection.JavaConversions.asScalaBuffer -import scala.collection.JavaConversions.bufferAsJavaList -import scala.collection.mutable -import scala.collection.mutable.ArrayBuffer - - -import org.apache.atlas.gremlin.GremlinExpressionFactory -import org.apache.atlas.gremlin.optimizer.GremlinQueryOptimizer -import org.apache.atlas.groovy.CastExpression -import org.apache.atlas.groovy.ClosureExpression -import org.apache.atlas.groovy.LabeledExpression -import org.apache.atlas.groovy.FunctionCallExpression -import org.apache.atlas.groovy.GroovyExpression -import org.apache.atlas.groovy.GroovyGenerationContext -import org.apache.atlas.groovy.IdentifierExpression -import org.apache.atlas.groovy.ListExpression -import org.apache.atlas.groovy.LiteralExpression -import org.apache.atlas.groovy.TraversalStepType -import org.apache.atlas.query.Expressions.AliasExpression -import org.apache.atlas.query.Expressions.ArithmeticExpression -import org.apache.atlas.query.Expressions.BackReference -import org.apache.atlas.query.Expressions.ClassExpression -import org.apache.atlas.query.Expressions.ComparisonExpression -import org.apache.atlas.query.Expressions.Expression -import org.apache.atlas.query.Expressions.ExpressionException -import org.apache.atlas.query.Expressions.FieldExpression -import org.apache.atlas.query.Expressions.FilterExpression -import org.apache.atlas.query.Expressions.InstanceExpression -import org.apache.atlas.query.Expressions.LimitExpression -import org.apache.atlas.query.Expressions.ListLiteral -import org.apache.atlas.query.Expressions.Literal -import org.apache.atlas.query.Expressions.LogicalExpression -import org.apache.atlas.query.Expressions.LoopExpression -import org.apache.atlas.query.Expressions.OrderExpression -import org.apache.atlas.query.Expressions.PathExpression -import org.apache.atlas.query.Expressions.SelectExpression -import org.apache.atlas.query.Expressions.TraitExpression -import org.apache.atlas.query.Expressions.TraitInstanceExpression -import org.apache.atlas.query.Expressions.hasFieldLeafExpression -import org.apache.atlas.query.Expressions.hasFieldUnaryExpression -import org.apache.atlas.query.Expressions.id -import org.apache.atlas.query.Expressions.isTraitLeafExpression -import org.apache.atlas.query.Expressions.isTraitUnaryExpression -import org.apache.atlas.repository.RepositoryException -import org.apache.atlas.repository.graphdb.AtlasEdgeDirection -import org.apache.atlas.typesystem.types.DataTypes -import org.apache.atlas.typesystem.types.DataTypes.TypeCategory -import org.apache.atlas.typesystem.types.IDataType -import org.apache.atlas.typesystem.types.TypeSystem -import org.joda.time.format.ISODateTimeFormat -import org.apache.atlas.query.Expressions.GroupByExpression -import org.apache.atlas.query.Expressions.MaxExpression -import org.apache.atlas.query.Expressions.MinExpression -import org.apache.atlas.query.Expressions.SumExpression -import org.apache.atlas.query.Expressions.CountExpression - -import org.apache.atlas.util.AtlasRepositoryConfiguration -import java.util.HashSet - -trait IntSequence { - def next: Int -} - -case class GremlinQuery(expr: Expression, queryStr: String, resultMaping: Map[String, (String, Int)]) { - - def hasSelectList = resultMaping != null - - def isPathExpression = expr.isInstanceOf[PathExpression] - - def isGroupBy = expr.isInstanceOf[GroupByExpression] -} - - -trait SelectExpressionHandling { - - class AliasFinder extends PartialFunction[Expression,Unit] { - val aliases = new HashSet[String]() - - def isDefinedAt(e: Expression) = true - - def apply(e: Expression) = e match { - case e@AliasExpression(_, alias) => { - aliases.add(alias) - } - case x => Unit - } - } - - class ReplaceAliasWithBackReference(aliases: HashSet[String]) extends PartialFunction[Expression, Expression] { - - def isDefinedAt(e: Expression) = true - - def apply(e: Expression) = e match { - case e@AliasExpression(child,alias) if aliases.contains(alias) => { - new BackReference(alias, child, None) - } - case x => x - } - } - - // Removes back references in comparison expressions that are - // right after an alias expression. - // - //For example: - // .as('x').and(select('x').has(y),...) is changed to - // .as('x').and(has(y),...) - // - //This allows the "has" to be extracted out of the and/or by - //the GremlinQueryOptimizer so the index can be used to evaluate - //the predicate. - - val RemoveUnneededBackReferences : PartialFunction[Expression, Expression] = { - - case filterExpr@FilterExpression(aliasExpr@AliasExpression(_,aliasName), filterChild) => { - val updatedChild = removeUnneededBackReferences(filterChild, aliasName) - val changed = !(updatedChild eq filterChild) - if(changed) { - FilterExpression(aliasExpr, updatedChild) - } - else { - filterExpr - } - - } - case x => x - } - def removeUnneededBackReferences(expr: Expression, outerAlias: String) : Expression = expr match { - case logicalExpr@LogicalExpression(logicalOp,children) => { - var changed : Boolean = false; - val updatedChildren : List[Expression] = children.map { child => - val updatedChild = removeUnneededBackReferences(child, outerAlias); - changed |= ! (updatedChild eq child); - updatedChild - } - if(changed) { - LogicalExpression(logicalOp,updatedChildren) - } - else { - logicalExpr - } - } - case comparisonExpr@ComparisonExpression(_,_,_) => { - var changed = false - val updatedLeft = removeUnneededBackReferences(comparisonExpr.left, outerAlias); - changed |= !( updatedLeft eq comparisonExpr.left); - - val updatedRight = removeUnneededBackReferences(comparisonExpr.right, outerAlias); - changed |= !(updatedRight eq comparisonExpr.right); - - if (changed) { - ComparisonExpression(comparisonExpr.symbol, updatedLeft, updatedRight) - } else { - comparisonExpr - } - } - case FieldExpression(fieldName, fieldInfo, Some(br @ BackReference(brAlias, _, _))) if outerAlias.equals(brAlias) => { - //Remove the back reference, since the thing it references is right in front - //of the comparison expression we're in - FieldExpression(fieldName, fieldInfo, None) - } - case x => x - } - - //in groupby, convert alias expressions defined in the group by child to BackReferences - //in the groupby list and selectList. - val AddBackReferencesToGroupBy : PartialFunction[Expression, Expression] = { - case GroupByExpression(child, groupBy, selExpr) => { - - val aliases = ArrayBuffer[AliasExpression]() - val finder = new AliasFinder(); - child.traverseUp(finder); - - val replacer = new ReplaceAliasWithBackReference(finder.aliases) - - val newGroupBy = new SelectExpression( - groupBy.child.transformUp(replacer), - groupBy.selectList.map { - expr => expr.transformUp(replacer) - - }, - groupBy.forGroupBy); - - val newSelExpr = new SelectExpression( - selExpr.child.transformUp(replacer), - selExpr.selectList.map { - expr => expr.transformUp(replacer) - }, - selExpr.forGroupBy); - - new GroupByExpression(child, newGroupBy, newSelExpr) - } - case x => x - } - /** - * To aide in gremlinQuery generation add an alias to the input of SelectExpressions - */ - class AddAliasToSelectInput extends PartialFunction[Expression, Expression] { - - private var idx = 0 - - def isDefinedAt(e: Expression) = true - - class DecorateFieldWithAlias(aliasE: AliasExpression) - extends PartialFunction[Expression, Expression] { - def isDefinedAt(e: Expression) = true - - def apply(e: Expression) = e match { - case fe@FieldExpression(fieldName, fInfo, None) => - FieldExpression(fieldName, fInfo, Some(BackReference(aliasE.alias, aliasE.child, None))) - case _ => e - } - } - - def apply(e: Expression) = e match { - case SelectExpression(aliasE@AliasExpression(_, _), selList, forGroupBy) => { - idx = idx + 1 - SelectExpression(aliasE, selList.map(_.transformUp(new DecorateFieldWithAlias(aliasE))), forGroupBy) - } - case SelectExpression(child, selList, forGroupBy) => { - idx = idx + 1 - val aliasE = AliasExpression(child, s"_src$idx") - SelectExpression(aliasE, selList.map(_.transformUp(new DecorateFieldWithAlias(aliasE))), forGroupBy) - } - case OrderExpression(aliasE@AliasExpression(_, _), order, asc) => { - OrderExpression(aliasE, order.transformUp(new DecorateFieldWithAlias(aliasE)),asc) - } - case OrderExpression(child, order, asc) => { - idx = idx + 1 - val aliasE = AliasExpression(child, s"_src$idx") - OrderExpression(aliasE, order.transformUp(new DecorateFieldWithAlias(aliasE)),asc) - } - case _ => e - } - } - - def getSelectExpressionSrc(e: Expression): List[String] = { - val l = ArrayBuffer[String]() - e.traverseUp { - case BackReference(alias, _, _) => l += alias - case ClassExpression(clsName) => l += clsName - } - l.toSet.toList - } - - def validateSelectExprHaveOneSrc: PartialFunction[Expression, Unit] = { - case SelectExpression(_, selList, forGroupBy) => { - selList.foreach { se => - val srcs = getSelectExpressionSrc(se) - if (srcs.size > 1) { - throw new GremlinTranslationException(se, "Only one src allowed in a Select Expression") - } - } - } - } - - def groupSelectExpressionsBySrc(sel: SelectExpression): mutable.LinkedHashMap[String, List[Expression]] = { - val m = mutable.LinkedHashMap[String, List[Expression]]() - sel.selectListWithAlias.foreach { se => - val l = getSelectExpressionSrc(se.child) - if (!m.contains(l(0))) { - m(l(0)) = List() - } - m(l(0)) = m(l(0)) :+ se.child - } - m - } - - /** - * For each Output Column in the SelectExpression compute the ArrayList(Src) this maps to and the position within - * this list. - * - * @param sel - * @return - */ - def buildResultMapping(sel: SelectExpression): Map[String, (String, Int)] = { - val srcToExprs = groupSelectExpressionsBySrc(sel) - val m = new mutable.HashMap[String, (String, Int)] - sel.selectListWithAlias.foreach { se => - val src = getSelectExpressionSrc(se.child)(0) - val srcExprs = srcToExprs(src) - var idx = srcExprs.indexOf(se.child) - m(se.alias) = (src, idx) - } - m.toMap - } -} - -class GremlinTranslationException(expr: Expression, reason: String) extends -ExpressionException(expr, s"Unsupported Gremlin translation: $reason") - -class GremlinTranslator(expr: Expression, - gPersistenceBehavior: GraphPersistenceStrategies) - extends SelectExpressionHandling { - - val preStatements = ArrayBuffer[GroovyExpression]() - val postStatements = ArrayBuffer[GroovyExpression]() - - val wrapAndRule: PartialFunction[Expression, Expression] = { - case f: FilterExpression if ((!f.condExpr.isInstanceOf[LogicalExpression]) && - (f.condExpr.isInstanceOf[isTraitLeafExpression] || !f.namedExpressions.isEmpty)) => - FilterExpression(f.child, new LogicalExpression("and", List(f.condExpr))) - } - - val validateComparisonForm: PartialFunction[Expression, Unit] = { - case c@ComparisonExpression(op, left, right) => - if (!left.isInstanceOf[FieldExpression]) { - throw new GremlinTranslationException(c, s"lhs of comparison is not a field") - } - if (!right.isInstanceOf[Literal[_]] && !right.isInstanceOf[ListLiteral[_]]) { - throw new GremlinTranslationException(c, - s"rhs of comparison is not a literal") - } - - if(right.isInstanceOf[ListLiteral[_]] && (!op.equals("=") && !op.equals("!="))) { - throw new GremlinTranslationException(c, - s"operation not supported with list literal") - } - () - } - - val counter = new IntSequence { - var i: Int = -1; - - def next: Int = { - i += 1; i - } - } - - def addAliasToLoopInput(c: IntSequence = counter): PartialFunction[Expression, Expression] = { - case l@LoopExpression(aliasE@AliasExpression(_, _), _, _) => l - case l@LoopExpression(inputExpr, loopExpr, t) => { - val aliasE = AliasExpression(inputExpr, s"_loop${c.next}") - LoopExpression(aliasE, loopExpr, t) - } - } - - def instanceClauseToTop(topE : Expression) : PartialFunction[Expression, Expression] = { - case le : LogicalExpression if (le fastEquals topE) => { - le.instance() - } - case ce : ComparisonExpression if (ce fastEquals topE) => { - ce.instance() - } - case he : hasFieldUnaryExpression if (he fastEquals topE) => { - he.instance() - } - } - - def traitClauseWithInstanceForTop(topE : Expression) : PartialFunction[Expression, Expression] = { -// This topE check prevented the comparison of trait expression when it is a child. Like trait as t limit 2 - case te : TraitExpression => { - val theTrait = te.as("theTrait") - val theInstance = theTrait.traitInstance().as("theInstance") - val outE = - theInstance.select(id("theTrait").as("traitDetails"), - id("theInstance").as("instanceInfo")) - QueryProcessor.validate(outE) - } - } - - def typeTestExpression(parent: GroovyExpression, typeName : String) : GroovyExpression = { - val stats = GremlinExpressionFactory.INSTANCE.generateTypeTestExpression(gPersistenceBehavior, parent, typeName, counter) - - preStatements ++= stats.init - stats.last - - } - - val QUOTE = "\""; - - private def cleanStringLiteral(l : Literal[_]) : String = { - return l.toString.stripPrefix(QUOTE).stripSuffix(QUOTE); - } - - - private def genQuery(parent: GroovyExpression, expr: Expression, inClosure: Boolean): GroovyExpression = expr match { - case ClassExpression(clsName) => typeTestExpression(parent, clsName) - case TraitExpression(clsName) => typeTestExpression(parent, clsName) - case fe@FieldExpression(fieldName, fInfo, child) - if fe.dataType.getTypeCategory == TypeCategory.PRIMITIVE || fe.dataType.getTypeCategory == TypeCategory.ARRAY => { - val fN = gPersistenceBehavior.fieldNameInVertex(fInfo.dataType, fInfo.attrInfo) - val childExpr = translateOptChild(parent, child, inClosure); - return GremlinExpressionFactory.INSTANCE.generateFieldExpression(childExpr, fInfo, fN, inClosure); - } - case fe@FieldExpression(fieldName, fInfo, child) - if fe.dataType.getTypeCategory == TypeCategory.CLASS || fe.dataType.getTypeCategory == TypeCategory.STRUCT => { - val childExpr = translateOptChild(parent, child, inClosure); - val direction = if (fInfo.isReverse) AtlasEdgeDirection.IN else AtlasEdgeDirection.OUT - val edgeLbl = gPersistenceBehavior.edgeLabel(fInfo) - return GremlinExpressionFactory.INSTANCE.generateAdjacentVerticesExpression(childExpr, direction, edgeLbl) - - } - case fe@FieldExpression(fieldName, fInfo, child) if fInfo.traitName != null => { - val childExpr = translateOptChild(parent, child, inClosure); - val direction = gPersistenceBehavior.instanceToTraitEdgeDirection - val edgeLbl = gPersistenceBehavior.edgeLabel(fInfo) - return GremlinExpressionFactory.INSTANCE.generateAdjacentVerticesExpression(childExpr, direction, edgeLbl) - - } - case c@ComparisonExpression(symb, f@FieldExpression(fieldName, fInfo, ch), l) => { - val qualifiedPropertyName = s"${gPersistenceBehavior.fieldNameInVertex(fInfo.dataType, fInfo.attrInfo)}" - - val childExpr = translateOptChild(parent, ch, inClosure) - val persistentExprValue : GroovyExpression = if(l.isInstanceOf[Literal[_]]) { - translateLiteralValue(fInfo.attrInfo.dataType, l.asInstanceOf[Literal[_]]); - } - else { - genQuery(null, l, inClosure); - } - - if (symb == "like") { - return GremlinExpressionFactory.INSTANCE.generateLikeExpressionUsingFilter(childExpr, qualifiedPropertyName, persistentExprValue); - } - - return GremlinExpressionFactory.INSTANCE.generateHasExpression(gPersistenceBehavior, childExpr, qualifiedPropertyName, c.symbol, persistentExprValue, fInfo); - } - case fil@FilterExpression(child, condExpr) => { - var newParent = genQuery(parent, child, inClosure); - val alias = "a" + counter.next; - newParent = GremlinExpressionFactory.INSTANCE.generateAliasExpression(newParent, alias); - val translated = genQuery(newParent, condExpr, inClosure); - //we want the query to return instances of the class whose instances we are filtering out - //The act of filtering may traverse edges and have other side effects. - GremlinExpressionFactory.INSTANCE.generateBackReferenceExpression(translated, false, alias); - } - case l@LogicalExpression(symb, children) => { - val translatedChildren : java.util.List[GroovyExpression] = translateList(children, false); - return GremlinExpressionFactory.INSTANCE.generateLogicalExpression(parent, symb, translatedChildren); - } - case sel@SelectExpression(child, selList, forGroupBy) => { - val m = groupSelectExpressionsBySrc(sel) - var srcNamesList: java.util.List[LiteralExpression] = new ArrayList() - var srcExprsList: List[java.util.List[GroovyExpression]] = List() - val it = m.iterator - - while (it.hasNext) { - val (src, selExprs) = it.next - srcNamesList.add(new LiteralExpression(src)); - val translatedSelExprs : java.util.List[GroovyExpression] = translateList(selExprs, true); - srcExprsList = srcExprsList :+ translatedSelExprs - } - val srcExprsStringList : java.util.List[GroovyExpression] = new ArrayList(); - srcExprsList.foreach { it => - srcExprsStringList.add(new ListExpression(it)); - } - - val childExpr = genQuery(parent, child, inClosure) - return GremlinExpressionFactory.INSTANCE.generateSelectExpression(childExpr, srcNamesList, srcExprsStringList); - - } - case loop@LoopExpression(input, loopExpr, t) => { - - val times : Integer = if(t.isDefined) { - t.get.rawValue.asInstanceOf[Integer] - } - else { - null.asInstanceOf[Integer] - } - val alias = input.asInstanceOf[AliasExpression].alias; - val inputQry = genQuery(parent, input, inClosure) - val translatedLoopExpr = genQuery(GremlinExpressionFactory.INSTANCE.getLoopExpressionParent(inputQry), loopExpr, inClosure); - return GremlinExpressionFactory.INSTANCE.generateLoopExpression(inputQry, gPersistenceBehavior, input.dataType, translatedLoopExpr, alias, times); - } - case BackReference(alias, _, _) => { - - return GremlinExpressionFactory.INSTANCE.generateBackReferenceExpression(parent, inClosure, alias); - } - case AliasExpression(child, alias) => { - var childExpr = genQuery(parent, child, inClosure); - return GremlinExpressionFactory.INSTANCE.generateAliasExpression(childExpr, alias); - } - case isTraitLeafExpression(traitName, Some(clsExp)) => { - val label = gPersistenceBehavior.traitLabel(clsExp.dataType, traitName); - return GremlinExpressionFactory.INSTANCE.generateAdjacentVerticesExpression(parent, AtlasEdgeDirection.OUT, label); - } - case isTraitUnaryExpression(traitName, child) => { - val label = gPersistenceBehavior.traitLabel(child.dataType, traitName); - return GremlinExpressionFactory.INSTANCE.generateAdjacentVerticesExpression(parent, AtlasEdgeDirection.OUT, label); - } - case hasFieldLeafExpression(fieldName, clsExp) => clsExp match { - case None => GremlinExpressionFactory.INSTANCE.generateUnaryHasExpression(parent, fieldName) - case Some(x) => { - val fi = TypeUtils.resolveReference(clsExp.get.dataType, fieldName); - if(! fi.isDefined) { - return GremlinExpressionFactory.INSTANCE.generateUnaryHasExpression(parent, fieldName); - } - else { - val fName = gPersistenceBehavior.fieldNameInVertex(fi.get.dataType, fi.get.attrInfo) - return GremlinExpressionFactory.INSTANCE.generateUnaryHasExpression(parent, fName); - } - } - } - case hasFieldUnaryExpression(fieldName, child) => - val childExpr = genQuery(parent, child, inClosure); - return GremlinExpressionFactory.INSTANCE.generateUnaryHasExpression(childExpr, fieldName); - case ArithmeticExpression(symb, left, right) => { - val leftExpr = genQuery(parent, left, inClosure); - val rightExpr = genQuery(parent, right, inClosure); - return GremlinExpressionFactory.INSTANCE.generateArithmeticExpression(leftExpr, symb, rightExpr); - } - case l: Literal[_] => { - - if(parent != null) { - return new org.apache.atlas.groovy.FieldExpression(parent, cleanStringLiteral(l)); - } - return translateLiteralValue(l.dataType, l); - } - case list: ListLiteral[_] => { - //Here, we are creating a Groovy list literal expression ([value1, value2, value3]). Because - //of this, any gremlin query expressions within the list must start with an anonymous traversal. - //We set 'inClosure' to true in this case to make that happen. - val values : java.util.List[GroovyExpression] = translateList(list.rawValue, true); - return new ListExpression(values); - } - case in@TraitInstanceExpression(child) => { - val childExpr = genQuery(parent, child, inClosure); - val direction = gPersistenceBehavior.traitToInstanceEdgeDirection; - return GremlinExpressionFactory.INSTANCE.generateAdjacentVerticesExpression(childExpr, direction); - } - case in@InstanceExpression(child) => { - return genQuery(parent, child, inClosure); - } - case pe@PathExpression(child) => { - val childExpr = genQuery(parent, child, inClosure) - return GremlinExpressionFactory.INSTANCE.generatePathExpression(childExpr); - } - case order@OrderExpression(child, odr, asc) => { - var orderExpression = odr - if(odr.isInstanceOf[BackReference]) { - orderExpression = odr.asInstanceOf[BackReference].reference - } - else if (odr.isInstanceOf[AliasExpression]) { - orderExpression = odr.asInstanceOf[AliasExpression].child - } - - val childExpr = genQuery(parent, child, inClosure); - var orderByParents : java.util.List[GroovyExpression] = GremlinExpressionFactory.INSTANCE.getOrderFieldParents(); - - val translatedParents : java.util.List[GroovyExpression] = new ArrayList[GroovyExpression](); - var translatedOrderParents = orderByParents.foreach { it => - translatedParents.add(genQuery(it, orderExpression, true)); - } - return GremlinExpressionFactory.INSTANCE.generateOrderByExpression(childExpr, translatedParents,asc); - - } - case limitOffset@LimitExpression(child, limit, offset) => { - val childExpr = genQuery(parent, child, inClosure); - val totalResultRows = limit.value + offset.value; - return GremlinExpressionFactory.INSTANCE.generateRangeExpression(childExpr, offset.value, totalResultRows); - } - case count@CountExpression() => { - val listExpr = GremlinExpressionFactory.INSTANCE.getClosureArgumentValue(); - GremlinExpressionFactory.INSTANCE.generateCountExpression(listExpr); - } - case max@MaxExpression(child) => { - //use "it" as the parent since the child will become - //part of a closure. Its value will be whatever vertex - //we are looking at in the collection. - val childExprParent = null; - val childExpr = genQuery(childExprParent, child, true); - val listExpr = GremlinExpressionFactory.INSTANCE.getClosureArgumentValue(); - GremlinExpressionFactory.INSTANCE.generateMaxExpression(listExpr, childExpr); - } - case min@MinExpression(child) => { - //use "it" as the parent since the child will become - //part of a closure. Its value will be whatever vertex - //we are looking at in the collection. - val childExprParent = null; - val childExpr = genQuery(childExprParent, child, true); - val listExpr = GremlinExpressionFactory.INSTANCE.getClosureArgumentValue(); - GremlinExpressionFactory.INSTANCE.generateMinExpression(listExpr, childExpr); - } - case sum@SumExpression(child) => { - //use "it" as the parent since the child will become - //part of a closure. Its value will be whatever vertex - //we are looking at in the collection. - val childExprParent = null; - val childExpr = genQuery(childExprParent, child, true); - val listExpr = GremlinExpressionFactory.INSTANCE.getClosureArgumentValue(); - GremlinExpressionFactory.INSTANCE.generateSumExpression(listExpr, childExpr); - } - case groupByExpr@GroupByExpression(child, groupBy, selExpr) => { - //remove aliases - val groupByExprListToTranslate = (groupBy.asInstanceOf[SelectExpression]).selectListWithAlias.map { - x => x.child; - } - val grpByExprsList = translateList(groupByExprListToTranslate, true); - val groupByValue = new ListExpression(grpByExprsList); - - //reduction only aggregate methods are supported here as of now.(Max, Min, Count) - //remove aliases - val srcExprListToTranslate = selExpr.selectListWithAlias.map { - x => x.child; - } - val srcExprsList = translateList(srcExprListToTranslate, true, true); - val srcExprsStringList = new ListExpression(srcExprsList) - - val childExpr = genQuery(parent, child, inClosure); - return GremlinExpressionFactory.INSTANCE.generateGroupByExpression(childExpr, groupByValue, srcExprsStringList); - } - case x => throw new GremlinTranslationException(x, "expression not yet supported") - } - - def translateList(exprs : List[Expressions.Expression], inClosure : Boolean, inGroupBy: Boolean = false) : java.util.List[GroovyExpression] = { - var parent = if (inGroupBy) { - GremlinExpressionFactory.INSTANCE.getGroupBySelectFieldParent(); - } - - else if(inClosure) { - null; - } - else { - GremlinExpressionFactory.INSTANCE.getAnonymousTraversalExpression() - } - var result : java.util.List[GroovyExpression] = new java.util.ArrayList(exprs.size); - exprs.foreach { it => - result.add(genQuery(parent, it, inClosure)); - } - return result; - } - - def translateOptChild(parent : GroovyExpression, child : Option[Expressions.Expression] , inClosure: Boolean) : GroovyExpression = child match { - - case Some(x) => genQuery(parent, x, inClosure) - case None => parent - } - - def translateLiteralValue(dataType: IDataType[_], l: Literal[_]): GroovyExpression = { - - - if (dataType == DataTypes.DATE_TYPE) { - try { - //Accepts both date, datetime formats - val dateStr = cleanStringLiteral(l) - val dateVal = ISODateTimeFormat.dateOptionalTimeParser().parseDateTime(dateStr).getMillis - return new LiteralExpression(dateVal) - } catch { - case pe: java.text.ParseException => - throw new GremlinTranslationException(l, - "Date format " + l + " not supported. Should be of the format " + - TypeSystem.getInstance().getDateFormat.toPattern); - } - } - else if(dataType == DataTypes.BYTE_TYPE) { - //cast needed, otherwise get class cast exception when trying to compare, since the - //persist value is assumed to be an Integer - return new CastExpression(new LiteralExpression(Byte.valueOf(s"""${l}"""), true),"byte"); - } - else if(dataType == DataTypes.INT_TYPE) { - return new LiteralExpression(Integer.valueOf(s"""${l}""")); - } - else if(dataType == DataTypes.BOOLEAN_TYPE) { - return new LiteralExpression(Boolean.valueOf(s"""${l}""")); - } - else if(dataType == DataTypes.SHORT_TYPE) { - return new CastExpression(new LiteralExpression(Short.valueOf(s"""${l}"""), true),"short"); - } - else if(dataType == DataTypes.LONG_TYPE) { - return new LiteralExpression(Long.valueOf(s"""${l}"""), true); - } - else if(dataType == DataTypes.FLOAT_TYPE) { - return new LiteralExpression(Float.valueOf(s"""${l}"""), true); - } - else if(dataType == DataTypes.DOUBLE_TYPE) { - return new LiteralExpression(Double.valueOf(s"""${l}"""), true); - } - else if(dataType == DataTypes.STRING_TYPE) { - return new LiteralExpression(cleanStringLiteral(l)); - } - else { - return new LiteralExpression(l.rawValue); - } - } - - def genFullQuery(expr: Expression, hasSelect: Boolean): String = { - - var q : GroovyExpression = new FunctionCallExpression(TraversalStepType.START, new IdentifierExpression(TraversalStepType.SOURCE, "g"),"V"); - - val debug:Boolean = false - if(gPersistenceBehavior.addGraphVertexPrefix(preStatements)) { - q = gPersistenceBehavior.addInitialQueryCondition(q); - } - - q = genQuery(q, expr, false) - - q = GremlinExpressionFactory.INSTANCE.generateToListExpression(q); - q = gPersistenceBehavior.getGraph().addOutputTransformationPredicate(q, hasSelect, expr.isInstanceOf[PathExpression]); - - - if(AtlasRepositoryConfiguration.isGremlinOptimizerEnabled()) { - q = GremlinQueryOptimizer.getInstance().optimize(q); - } - - val closureExpression = new ClosureExpression(); - - closureExpression.addStatements(preStatements); - closureExpression.addStatement(q) - closureExpression.addStatements(postStatements); - - val overallExpression = new LabeledExpression("L", closureExpression); - - val qryStr = generateGremlin(overallExpression); - - if(debug) { - println(" query " + qryStr) - } - - qryStr; - - } - - def generateGremlin(expr: GroovyExpression) : String = { - val ctx : GroovyGenerationContext = new GroovyGenerationContext(); - ctx.setParametersAllowed(false); - expr.generateGroovy(ctx); - return ctx.getQuery; - } - - def translate(): GremlinQuery = { - var e1 = expr.transformUp(wrapAndRule) - - e1.traverseUp(validateComparisonForm) - e1 = e1.transformUp(AddBackReferencesToGroupBy) - e1 = e1.transformUp(new AddAliasToSelectInput) - e1.traverseUp(validateSelectExprHaveOneSrc) - e1 = e1.transformUp(addAliasToLoopInput()) - e1 = e1.transformUp(instanceClauseToTop(e1)) - e1 = e1.transformUp(traitClauseWithInstanceForTop(e1)) - e1 = e1.transformUp(RemoveUnneededBackReferences) - - //Following code extracts the select expressions from expression tree. - - val se = SelectExpressionHelper.extractSelectExpression(e1) - if (se.isDefined) { - val rMap = buildResultMapping(se.get) - GremlinQuery(e1, genFullQuery(e1, true), rMap) - } else { - GremlinQuery(e1, genFullQuery(e1, false), null) - } - - } -} - object SelectExpressionHelper { - /** - * This method extracts the child select expression from parent expression - */ - def extractSelectExpression(child: Expression): Option[SelectExpression] = { - child match { - case se@SelectExpression(child, selectList, false) =>{ - Some(se) - } - case limit@LimitExpression(child, lmt, offset) => { - extractSelectExpression(child) - } - case order@OrderExpression(child, odr, odrBy) => { - extractSelectExpression(child) - } - case path@PathExpression(child) => { - extractSelectExpression(child) - } - case _ => { - None - } - - } - } - } - /* - * TODO - * Translation Issues: - * 1. back references in filters. For e.g. testBackreference: 'DB as db Table where (db.name = "Reporting")' - * this is translated to: - * g.V.has("typeName","DB").as("db").in("Table.db").and(_().back("db").has("name", T.eq, "Reporting")).map().toList() - * But the '_().back("db") within the and is ignored, the has condition is applied on the current element. - * The solution is to to do predicate pushdown and apply the filter immediately on top of the referred Expression. - */ -
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/scala/org/apache/atlas/query/QueryParser.scala ---------------------------------------------------------------------- diff --git a/repository/src/main/scala/org/apache/atlas/query/QueryParser.scala b/repository/src/main/scala/org/apache/atlas/query/QueryParser.scala deleted file mode 100755 index 4bc6e74..0000000 --- a/repository/src/main/scala/org/apache/atlas/query/QueryParser.scala +++ /dev/null @@ -1,557 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.query - -import org.apache.atlas.query.Expressions._ - -import scala.util.parsing.combinator.lexical.StdLexical -import scala.util.parsing.combinator.syntactical.StandardTokenParsers -import scala.util.parsing.combinator.{ImplicitConversions, PackratParsers} -import scala.util.parsing.input.CharArrayReader._ -import org.apache.atlas.AtlasException -import org.apache.atlas.typesystem.types.DataTypes - -trait QueryKeywords { - this: StandardTokenParsers => - - import scala.language.implicitConversions - - protected case class Keyword(str: String) - - protected implicit def asParser(k: Keyword): Parser[String] = k.str - - protected val LIST_LPAREN = Keyword("[") - protected val LIST_RPAREN = Keyword("]") - protected val LPAREN = Keyword("(") - protected val RPAREN = Keyword(")") - protected val EQ = Keyword("=") - protected val LT = Keyword("<") - protected val GT = Keyword(">") - protected val NEQ = Keyword("!=") - protected val LTE = Keyword("<=") - protected val GTE = Keyword(">=") - protected val COMMA = Keyword(",") - protected val AND = Keyword("and") - protected val OR = Keyword("or") - protected val PLUS = Keyword("+") - protected val MINUS = Keyword("-") - protected val STAR = Keyword("*") - protected val DIV = Keyword("/") - protected val DOT = Keyword(".") - - protected val SELECT = Keyword("select") - protected val FROM = Keyword("from") - protected val WHERE = Keyword("where") - protected val GROUPBY = Keyword("groupby") - protected val LOOP = Keyword("loop") - protected val ISA = Keyword("isa") - protected val IS = Keyword("is") - protected val HAS = Keyword("has") - protected val AS = Keyword("as") - protected val TIMES = Keyword("times") - protected val WITHPATH = Keyword("withPath") - protected val LIMIT = Keyword("limit") - protected val OFFSET = Keyword("offset") - protected val ORDERBY = Keyword("orderby") - protected val COUNT = Keyword("count") - protected val MAX = Keyword("max") - protected val MIN = Keyword("min") - protected val SUM = Keyword("sum") - protected val BY = Keyword("by") - protected val ORDER = Keyword("order") - protected val LIKE = Keyword("like") -} - -trait ExpressionUtils { - - protected val DESC = "desc" - def loop(input: Expression, l: (Expression, Option[Literal[Integer]], Option[String])) = l match { - case (c, None, None) => input.loop(c) - case (c, t, None) => input.loop(c, t.get) - case (c, None, Some(a)) => input.loop(c).as(a) - case (c, t, Some(a)) => input.loop(c, t.get).as(a) - } - - def select(input: Expression, s: List[(Expression, Option[String])], forGroupBy: Boolean = false) = { - val selList = s.map { t => - t._2 match { - case None => t._1.as(s"${t._1}") - case _ => t._1.as(t._2.get) - } - } - new SelectExpression(input, selList, forGroupBy) - } - - def limit(input: Expression, lmt: Literal[Integer], offset: Literal[Integer]) = { - input.limit(lmt, offset) - } - - def order(input: Expression, odr: Expression, asc: Boolean) = { - input.order(odr, asc) - } - - def leftmostId(e: Expression) = { - var le: IdExpression = null - e.traverseUp { case i: IdExpression if le == null => le = i} - le - } - - def notIdExpression = new PartialFunction[Expression, Expression] { - def isDefinedAt(x: Expression): Boolean = !x.isInstanceOf[IdExpression] - - def apply(e: Expression) = e - } - - def replaceIdWithField(id: IdExpression, fe: UnresolvedFieldExpression): PartialFunction[Expression, Expression] = { - case e: IdExpression if e == id => fe - } - - def merge(snglQuery1: Expression, sngQuery2: Expression): Expression = { - val leftSrcId = leftmostId(sngQuery2) - sngQuery2.transformUp(replaceIdWithField(leftSrcId, snglQuery1.field(leftSrcId.name))) - } - - def groupBy(input: Expression, groupByExpr: SelectExpression, selectExpr: SelectExpression) = { - input.groupBy(groupByExpr, selectExpr) - } -} - -case class QueryParams(limit: Int, offset: Int) - -/** - * Query parser is used to parse the DSL query. It uses scala PackratParsers and pattern matching to extract the expressions. - * It builds up a expression tree. - */ -object QueryParser extends StandardTokenParsers with QueryKeywords with ExpressionUtils with PackratParsers { - - import scala.language.higherKinds - - private val reservedWordsDelims: Seq[String] = this. - getClass.getMethods.filter(_.getReturnType == classOf[Keyword]).map(_.invoke(this).asInstanceOf[Keyword].str) - - private val (queryreservedWords: Seq[String], querydelims: Seq[String]) = - reservedWordsDelims.partition(s => s.charAt(0).isLetter) - - override val lexical = new QueryLexer(queryreservedWords, querydelims) - - /** - * @param input query string - * @param queryParams query parameters that contains limit and offset - * @return - */ - def apply(input: String)(implicit queryParams: QueryParams = null): Either[NoSuccess, Expression] = synchronized { - phrase(queryWithPath)(new lexical.Scanner(input)) match { - case Success(r, x) => Right(r) - case f@Failure(m, x) => Left(f) - case e@Error(m, x) => Left(e) - } - } - - import scala.math._ - - def queryWithPath(implicit queryParams: QueryParams) = query ~ opt(WITHPATH) ^^ { - case q ~ None => q - case q ~ p => q.path() - } - - /** - * A singleQuery can have the following forms: - * 1. SrcQuery [select] [orderby desc] [Limit x offset y] -> source query followed by optional select statement followed by optional order by followed by optional limit - * eg: Select "hive_db where hive_db has name orderby 'hive_db.owner' limit 2 offset 1" - * - * @return - */ - def query(implicit queryParams: QueryParams) = querySrc ~ opt(loopExpression) ~ opt(groupByExpr) ~ opt(selectClause) ~ opt(orderby) ~ opt(limitOffset) ^^ { - case s ~ l ~ grp ~ sel ~ odr ~ lmtoff => { - var expressiontree = s - if (l.isDefined) //Note: The order of if statements is important. - { - expressiontree = loop(expressiontree, l.get); - } - if (odr.isDefined) - { - expressiontree = order(expressiontree, odr.get._1, odr.get._2) - } - if (queryParams != null && lmtoff.isDefined) - { - val mylimit = int(min(queryParams.limit, max(lmtoff.get._1 - queryParams.offset, 0))) - val myoffset = int(queryParams.offset + lmtoff.get._2) - expressiontree = limit(expressiontree, mylimit, myoffset) - } else if(lmtoff.isDefined) { - expressiontree = limit(expressiontree, int(lmtoff.get._1), int(lmtoff.get._2)) - } else if(queryParams != null) { - expressiontree = limit(expressiontree, int(queryParams.limit), int(queryParams.offset)) - } - if (grp.isDefined && sel.isDefined) - { - - var child = expressiontree - var selectExpr: SelectExpression = select(child, sel.get, true) - var grpBySelectExpr: SelectExpression = select(child, grp.get, true) - expressiontree = groupBy(child, grpBySelectExpr, selectExpr) - } - else if (grp.isDefined) - { - throw new AtlasException("groupby without select is not allowed"); - } - else if (sel.isDefined) - { - var selectChild = expressiontree - val selExpr : SelectExpression = select(selectChild, sel.get); - if(selExpr.hasAggregation) { - //In order to do the aggregation, we need to add an implicit group by. Having the - //group by expression be a constant values forces all of the vertices into one group. - val groupByConstant : Expression = Expressions.literal(DataTypes.STRING_TYPE, "dummy"); - val groupBySelExpr : SelectExpression = select(selectChild, sel.get, true); - val groupByListExpr : SelectExpression = select(selectChild, List((groupByConstant,None)), true) - expressiontree = groupBy(selectChild, groupByListExpr, groupBySelExpr) - } - else { - expressiontree = selExpr - } - } - expressiontree - } - } - - def querySrc: Parser[Expression] = rep1sep(singleQrySrc, opt(COMMA)) ^^ { l => l match { - case h :: Nil => h - case h :: t => t.foldLeft(h)(merge(_, _)) - case Nil => null - } - } - - /** - * A SingleQuerySrc can have the following forms: - * 1. FROM id [WHERE] [expr] -> from optionally followed by a filter - * 2. WHERE expr -> where clause, FROM is assumed to be the leftmost Id in the where clause - * 3. expr (that is not an IdExpression) -> where clause, FROM is assumed to be the leftmost Id in the expr - * 4. Id [WHERE] [expr] -> from optionally followed by a filter - * - * @return - */ - def singleQrySrc: Parser[Expression] = FROM ~ fromSrc ~ opt(WHERE) ~ opt(expr ^? notIdExpression) ^^ { - case f ~ i ~ w ~ None => i - case f ~ i ~ w ~ c => i.where(c.get) - } | - WHERE ~ (expr ^? notIdExpression) ^^ { case w ~ e => { - val lId = leftmostId(e) - if (lId == null) { - failure("cannot infer Input from the where clause") - } - lId.where(e) - } - } | - expr ^? notIdExpression ^^ { case e => { - val lId = leftmostId(e) - if (lId == null) { - failure("cannot infer Input from the where clause") - } - lId.where(e) - } - } | - fromSrc ~ opt(WHERE) ~ opt(expr ^? notIdExpression) ^^ { - case i ~ w ~ None => i - case i ~ w ~ c => i.where(c.get) - } - - def fromSrc = identifier ~ AS ~ alias ^^ { case s ~ a ~ al => s.as(al)} | - identifier - - def orderby = (ORDERBY|(ORDER ~ BY )) ~ expr ~ opt (asce) ^^ { - case o ~ odr ~ None => (odr, true) - case o ~ odr ~ asc => (odr, asc.get) - } - - def limitOffset: Parser[(Int, Int)] = LIMIT ~ lmt ~ opt (offset) ^^ { - case l ~ lt ~ None => (lt.toInt, 0) - case l ~ lt ~ of => (lt.toInt, of.get.toInt) - } - - def offset = OFFSET ~ ofset ^^ { - case offset ~ of => of - } - - def asce = asc ^^ { - case DESC => false - case _ => true - } - - def loopExpression(implicit queryParams: QueryParams): Parser[(Expression, Option[Literal[Integer]], Option[String])] = - LOOP ~ (LPAREN ~> query <~ RPAREN) ~ opt(intConstant <~ TIMES) ~ opt(AS ~> alias) ^^ { - case l ~ e ~ None ~ a => (e, None, a) - case l ~ e ~ Some(i) ~ a => (e, Some(int(i)), a) - } - - def selectClause: Parser[List[(Expression, Option[String])]] = SELECT ~ rep1sep(selectExpression, COMMA) ^^ { - case s ~ cs => cs - } - def selectExpression: Parser[(Expression, Option[String])] = expr ~ opt(AS ~> alias) ^^ { - case e ~ a => (e, a) - } - - def expr: Parser[Expression] = compE ~ opt(rep(exprRight)) ^^ { - case l ~ None => l - case l ~ Some(r) => r.foldLeft(l) { (l, r) => l.logicalOp(r._1)(r._2)} - } - - def exprRight = (AND | OR) ~ compE ^^ { case op ~ c => (op, c)} - - def compE = - arithE ~ (LT | LTE | EQ | NEQ | GT | GTE | LIKE) ~ arithE ^^ { case l ~ op ~ r => l.compareOp(op)(r)} | - arithE ~ (ISA | IS) ~ ident ^^ { case l ~ i ~ t => l.isTrait(t)} | - arithE ~ HAS ~ ident ^^ { case l ~ i ~ f => l.hasField(f)} | - arithE | countClause | maxClause | minClause | sumClause - - def arithE = multiE ~ opt(rep(arithERight)) ^^ { - case l ~ None => l - case l ~ Some(r) => r.foldLeft(l) { (l, r) => l.arith(r._1)(r._2)} - } - - def arithERight = (PLUS | MINUS) ~ multiE ^^ { case op ~ r => (op, r)} - - def multiE = atomE ~ opt(rep(multiERight)) ^^ { - case l ~ None => l - case l ~ Some(r) => r.foldLeft(l) { (l, r) => l.arith(r._1)(r._2)} - } - - def multiERight = (STAR | DIV) ~ atomE ^^ { case op ~ r => (op, r)} - - def atomE = literal | identifier | LPAREN ~> expr <~ RPAREN | listLiteral - - def listLiteral = LIST_LPAREN ~ rep1sep(literal, COMMA) ~ LIST_RPAREN ^^ { - case lp ~ le ~ rp => list(le) - } - - def identifier = rep1sep(ident, DOT) ^^ { l => l match { - - /* - * We don't have enough context here to know what the id can be. - * Examples: - * Column isa PII - "Column" could be a field, type, or alias - * name = 'John' - "name" must be a field. - * Use generic id(), let type the be refined based on the context later. - */ - case h :: Nil => id(h) - - /* - * Then left-most part of the identifier ("h") must be a can be either. However, - * Atlas does support struct attributes, whose fields must accessed through - * this syntax. Let the downstream processing figure out which case we're in. - * - * Examples: - * hive_table.name - here, hive_table must be a type - * sortCol.order - here, sortCol is a struct attribute, must resolve to a field. - */ - case h :: t => { //the left-most part of the identifier (h) can be - t.foldLeft(id(h).asInstanceOf[Expression])(_.field(_)) - } - - case Nil => null - } - } - - def alias = ident | stringLit - - def lmt = intConstant - - def ofset = intConstant - - def asc = ident | stringLit - - def literal = booleanConstant ^^ { - boolean(_) - } | - intConstant ^^ { - int(_) - } | - longConstant ^^ { - long(_) - } | - floatConstant ^^ { - float(_) - } | - doubleConstant ^^ { - double(_) - } | - stringLit ^^ { - string(_) - } - - def booleanConstant: Parser[String] = - elem("int", _.isInstanceOf[lexical.BooleanLiteral]) ^^ (_.chars) - - def intConstant: Parser[String] = - elem("int", _.isInstanceOf[lexical.IntLiteral]) ^^ (_.chars) - - def longConstant: Parser[String] = - elem("int", _.isInstanceOf[lexical.LongLiteral]) ^^ (_.chars) - - def floatConstant: Parser[String] = - elem("int", _.isInstanceOf[lexical.FloatLiteral]) ^^ (_.chars) - - def doubleConstant: Parser[String] = - elem("int", _.isInstanceOf[lexical.DoubleLiteral]) ^^ (_.chars) - - def countClause = COUNT ~ LPAREN ~ RPAREN ^^ { - case c => count() - } - def maxClause = MAX ~ (LPAREN ~> expr <~ RPAREN) ^^ { - case m ~ e => maxExpr(e) - } - def minClause = MIN ~ (LPAREN ~> expr <~ RPAREN) ^^ { - case m ~ e => minExpr(e) - } - def sumClause = SUM ~ (LPAREN ~> expr <~ RPAREN) ^^ { - case m ~ e => sumExpr(e) - } - def groupByExpr = GROUPBY ~ (LPAREN ~> rep1sep(selectExpression, COMMA) <~ RPAREN) ^^ { - case g ~ ce => ce - } - - def isKeyword(s: String) = queryreservedWords.contains(s) -} - -class QueryLexer(val keywords: Seq[String], val delims: Seq[String]) extends StdLexical with ImplicitConversions { - - case class BooleanLiteral(chars: String) extends Token { - override def toString = chars - } - - case class IntLiteral(chars: String) extends Token { - override def toString = chars - } - - case class LongLiteral(chars: String) extends Token { - override def toString = chars - } - - case class FloatLiteral(chars: String) extends Token { - override def toString = chars - } - - case class DoubleLiteral(chars: String) extends Token { - override def toString = chars - } - - reserved ++= keywords.flatMap(w => allCaseVersions(w)) - - delimiters ++= delims - - override lazy val token: Parser[Token] = - ( - (trueP | falseP) - | longConstant ^^ LongLiteral - | intConstant ^^ IntLiteral - | floatConstant ^^ FloatLiteral - | dubConstant ^^ DoubleLiteral - | identifier ^^ processIdent - | quotedIdentifier ^^ Identifier - | string ^^ StringLit - | EofCh ^^^ EOF - | '\'' ~> failure("unclosed string literal") - | '"' ~> failure("unclosed string literal") - | delim - | '.' ^^^ new Keyword(".") - | failure("illegal character") - ) - - override def identChar = letter | elem('_') - - def identifier = identChar ~ (identChar | digit).* ^^ { case first ~ rest => (first :: rest).mkString} - - def quotedIdentifier = '`' ~> chrExcept('`', '\n', EofCh).* <~ '`' ^^ { - _ mkString "" - } - - override def whitespace: Parser[Any] = - (whitespaceChar - | '/' ~ '*' ~ comment - | '/' ~ '/' ~ chrExcept(EofCh, '\n').* - | '#' ~ chrExcept(EofCh, '\n').* - | '/' ~ '*' ~ failure("unclosed comment") - ).* - - protected override def comment: Parser[Any] = ( - commentChar.* ~ '*' ~ '/' - ) - - protected def commentChar = chrExcept(EofCh, '*') | '*' ~ not('/') - - def string = '\"' ~> chrExcept('\"', '\n', EofCh).* <~ '\"' ^^ { - _ mkString "" - } | - '\'' ~> chrExcept('\'', '\n', EofCh).* <~ '\'' ^^ { - _ mkString "" - } - - def zero: Parser[String] = '0' ^^^ "0" - - def nonzero = elem("nonzero digit", d => d.isDigit && d != '0') - - def sign = elem("sign character", d => d == '-' || d == '+') - - def exponent = elem("exponent character", d => d == 'e' || d == 'E') - - - def intConstant = opt(sign) ~> zero | intList - - def intList = opt(sign) ~ nonzero ~ rep(digit) ^^ { case s ~ x ~ y => (optString("", s) :: x :: y) mkString ""} - - def fracPart: Parser[String] = '.' ~> rep(digit) ^^ { r => - "." + (r mkString "") - } - - def expPart = exponent ~ opt(sign) ~ rep1(digit) ^^ { case e ~ s ~ d => - e.toString + optString("", s) + d.mkString("") - } - - def dubConstant = opt(sign) ~ digit.+ ~ fracPart ~ opt(expPart) ^^ { - case s ~ i ~ f ~ e => { - optString("", s) + (i mkString "") + f + optString("", e) - } - } - - def floatConstant = opt(sign) ~ digit.* ~ fracPart ~ 'f' ^^ { case s ~ i ~ fr ~ f => - optString("", s) + i + fr - } | opt(sign) ~ digit.+ ~ opt(fracPart) ~ 'f' ^^ { case s ~ i ~ fr ~ f => - optString("", s) + i + optString("", fr) - } - - def longConstant = intConstant ~ 'l' ^^ { case i ~ l => i} - - def trueP = 't' ~ 'r' ~ 'u' ~ 'e' ^^^ BooleanLiteral("true") - - def falseP = 'f' ~ 'a' ~ 'l' ~ 's' ~ 'e' ^^^ BooleanLiteral("false") - - private def optString[A](pre: String, a: Option[A]) = a match { - case Some(x) => pre + x.toString - case None => "" - } - - /** Generate all variations of upper and lower case of a given string */ - def allCaseVersions(s: String, prefix: String = ""): Stream[String] = { - if (s.isEmpty) { - Stream(prefix) - } else { - allCaseVersions(s.tail, prefix + s.head.toLower) #::: - allCaseVersions(s.tail, prefix + s.head.toUpper) - } - } -} http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/scala/org/apache/atlas/query/QueryProcessor.scala ---------------------------------------------------------------------- diff --git a/repository/src/main/scala/org/apache/atlas/query/QueryProcessor.scala b/repository/src/main/scala/org/apache/atlas/query/QueryProcessor.scala deleted file mode 100755 index e1e8408..0000000 --- a/repository/src/main/scala/org/apache/atlas/query/QueryProcessor.scala +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.query - -import org.apache.atlas.repository.graphdb.AtlasGraph -import org.apache.atlas.query.Expressions._ -import org.slf4j.{Logger, LoggerFactory} -import org.apache.atlas.util.AtlasRepositoryConfiguration -import org.apache.atlas.utils.LruCache -import org.apache.atlas.util.CompiledQueryCacheKey -import java.util.Collections - -object QueryProcessor { - val LOG : Logger = LoggerFactory.getLogger("org.apache.atlas.query.QueryProcessor") - - val compiledQueryCache = Collections.synchronizedMap(new LruCache[CompiledQueryCacheKey, GremlinQuery]( - AtlasRepositoryConfiguration.getCompiledQueryCacheCapacity(), - AtlasRepositoryConfiguration.getCompiledQueryCacheEvictionWarningThrottle())); - - def evaluate(e: Expression, g: AtlasGraph[_,_], gP : GraphPersistenceStrategies = null): - GremlinQueryResult = { - - var strategy = gP; - if(strategy == null) { - strategy = GraphPersistenceStrategy1(g); - } - - //convert the query expression to DSL so we can check whether or not it is in the compiled - //query cache and avoid validating/translating it again if it is. - val dsl = e.toString(); - val cacheKey = new CompiledQueryCacheKey(dsl); - var q = compiledQueryCache.get(cacheKey); - if(q == null) { - - //query was not found in the compiled query cache. Validate - //and translate it, then cache the result. - - val e1 = validate(e) - q = new GremlinTranslator(e1, strategy).translate() - compiledQueryCache.put(cacheKey, q); - if(LOG.isDebugEnabled()) { - LOG.debug("Validated Query: " + e1) - LOG.debug("Expression Tree:\n" + e1.treeString); - } - } - if(LOG.isDebugEnabled()) { - LOG.debug("DSL Query: " + dsl); - LOG.debug("Gremlin Query: " + q.queryStr) - } - new GremlinEvaluator(q, strategy, g).evaluate() - } - - def validate(e: Expression): Expression = { - - val e1 = e.transformUp(refineIdExpressionType); - val e2 = e1.transformUp(new Resolver(None,e1.namedExpressions)) - - e2.traverseUp { - case x: Expression if !x.resolved => - throw new ExpressionException(x, s"Failed to resolved expression $x") - } - - /* - * trigger computation of dataType of expression tree - */ - e2.dataType - - /* - * ensure fieldReferences match the input expression's dataType - */ - val e3 = e2.transformUp(FieldValidator) - val e4 = e3.transformUp(new Resolver(None,e3.namedExpressions)) - - e4.dataType - - e4 - } - - val convertToFieldIdExpression : PartialFunction[Expression,Expression] = { - case IdExpression(name, IdExpressionType.Unresolved) => IdExpression(name, IdExpressionType.NonType); - } - - - //this function is called in a depth first manner on the expression tree to set the exprType in IdExpressions - //when we know them. Since Expression classes are immutable, in order to do this we need to create new instances - //of the case. The logic here enumerates the cases that have been identified where the given IdExpression - //cannot resolve to a class or trait. This is the case in any places where a field value must be used. - //For example, you cannot add two classes together or compare traits. Any IdExpressions in those contexts - //refer to unqualified attribute names. On a similar note, select clauses need to product an actual value. - //For example, in 'from DB select name' or 'from DB select name as n', name must be an attribute. - val refineIdExpressionType : PartialFunction[Expression,Expression] = { - - //spit out the individual cases to minimize the object churn. Specifically, for ComparsionExpressions where neither - //child is an IdExpression, there is no need to create a new ComparsionExpression object since neither child will - //change. This applies to ArithmeticExpression as well. - case c@ComparisonExpression(symbol, l@IdExpression(_,IdExpressionType.Unresolved) , r@IdExpression(_,IdExpressionType.Unresolved)) => { - ComparisonExpression(symbol, convertToFieldIdExpression(l), convertToFieldIdExpression(r)) - } - case c@ComparisonExpression(symbol, l@IdExpression(_,IdExpressionType.Unresolved) , r) => ComparisonExpression(symbol, convertToFieldIdExpression(l), r) - case c@ComparisonExpression(symbol, l, r@IdExpression(_,IdExpressionType.Unresolved)) => ComparisonExpression(symbol, l, convertToFieldIdExpression(r)) - - case e@ArithmeticExpression(symbol, l@IdExpression(_,IdExpressionType.Unresolved) , r@IdExpression(_,IdExpressionType.Unresolved)) => { - ArithmeticExpression(symbol, convertToFieldIdExpression(l), convertToFieldIdExpression(r)) - } - case e@ArithmeticExpression(symbol, l@IdExpression(_,IdExpressionType.Unresolved) , r) => ArithmeticExpression(symbol, convertToFieldIdExpression(l), r) - case e@ArithmeticExpression(symbol, l, r@IdExpression(_,IdExpressionType.Unresolved)) => ArithmeticExpression(symbol, l, convertToFieldIdExpression(r)) - - case s@SelectExpression(child, selectList, forGroupBy) => { - var changed = false - val newSelectList = selectList.map { - - expr => expr match { - case e@IdExpression(_,IdExpressionType.Unresolved) => { changed=true; convertToFieldIdExpression(e) } - case AliasExpression(child@IdExpression(_,IdExpressionType.Unresolved), alias) => {changed=true; AliasExpression(convertToFieldIdExpression(child), alias)} - case x => x - } - } - if(changed) { - SelectExpression(child, newSelectList, forGroupBy) - } - else { - s - } - } - - } -} http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/scala/org/apache/atlas/query/Resolver.scala ---------------------------------------------------------------------- diff --git a/repository/src/main/scala/org/apache/atlas/query/Resolver.scala b/repository/src/main/scala/org/apache/atlas/query/Resolver.scala deleted file mode 100755 index 1b42f3e..0000000 --- a/repository/src/main/scala/org/apache/atlas/query/Resolver.scala +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.query - -import org.apache.atlas.query.Expressions._ -import org.apache.atlas.typesystem.types.IDataType -import org.apache.atlas.typesystem.types.TraitType -import org.apache.atlas.typesystem.types.ClassType -class Resolver(srcExpr: Option[Expression] = None, aliases: Map[String, Expression] = Map(), - connectClassExprToSrc: Boolean = false) - extends PartialFunction[Expression, Expression] { - - import org.apache.atlas.query.TypeUtils._ - - def isDefinedAt(x: Expression) = true - - def apply(e: Expression): Expression = e match { - case idE@IdExpression(name, exprType) => { - - val backExpr = aliases.get(name) - if (backExpr.isDefined) { - if(backExpr.get.resolved) { - return new BackReference(name, backExpr.get, None) - } - else { - //replace once resolved - return idE; - } - } - - if (srcExpr.isDefined) { - val fInfo = resolveReference(srcExpr.get.dataType, name) - if (fInfo.isDefined) { - return new FieldExpression(name, fInfo.get, None) - } - } - - if(exprType.isTypeAllowed) { - val dt = resolveAsDataType(name); - if(dt.isDefined) { - if(dt.get.isInstanceOf[ClassType]) { - return new ClassExpression(name) - } - if(dt.get.isInstanceOf[TraitType]) { - return new TraitExpression(name) - } - } - } - idE - } - case ce@ClassExpression(clsName) if connectClassExprToSrc && srcExpr.isDefined => { - val fInfo = resolveReference(srcExpr.get.dataType, clsName) - if (fInfo.isDefined) { - return new FieldExpression(clsName, fInfo.get, None) - } - ce - } - case f@UnresolvedFieldExpression(child, fieldName) if child.resolved => { - var fInfo: Option[FieldInfo] = None - - fInfo = resolveReference(child.dataType, fieldName) - if (fInfo.isDefined) { - return new FieldExpression(fieldName, fInfo.get, Some(child)) - } - val tType = resolveAsTraitType(fieldName) - if (tType.isDefined) { - return new FieldExpression(fieldName, FieldInfo(child.dataType, null, null, fieldName), Some(child)) - } - f - } - case isTraitLeafExpression(traitName, classExpression) - if srcExpr.isDefined && !classExpression.isDefined => - isTraitLeafExpression(traitName, srcExpr) - case hasFieldLeafExpression(traitName, classExpression) - if srcExpr.isDefined && !classExpression.isDefined => - hasFieldLeafExpression(traitName, srcExpr) - case f@FilterExpression(inputExpr, condExpr) if inputExpr.resolved => { - val r = new Resolver(Some(inputExpr), inputExpr.namedExpressions) - return new FilterExpression(inputExpr, condExpr.transformUp(r)) - } - case SelectExpression(child, selectList, forGroupBy) if child.resolved => { - val r = new Resolver(Some(child), child.namedExpressions) - return new SelectExpression(child, selectList.map { - _.transformUp(r) - }, forGroupBy) - } - case l@LoopExpression(inputExpr, loopExpr, t) if inputExpr.resolved => { - val r = new Resolver(Some(inputExpr), inputExpr.namedExpressions, true) - return new LoopExpression(inputExpr, loopExpr.transformUp(r), t) - } - case lmt@LimitExpression(child, limit, offset) => { - val r = new Resolver(Some(child), child.namedExpressions) - return new LimitExpression(child.transformUp(r), limit, offset) - } - case order@OrderExpression(child, odr, asc) => { - val r = new Resolver(Some(child), child.namedExpressions) - return new OrderExpression(child, odr.transformUp(r), asc) - } - case x => x - } -} - -/** - * - any FieldReferences that explicitly reference the input, can be converted to implicit references - * - any FieldReferences that explicitly reference a - */ -object FieldValidator extends PartialFunction[Expression, Expression] { - - def isDefinedAt(x: Expression) = true - - def isSrc(e: Expression) = e.isInstanceOf[ClassExpression] || e.isInstanceOf[TraitExpression] - - def validateQualifiedField(srcDataType: IDataType[_]): PartialFunction[Expression, Expression] = { - case FieldExpression(fNm, fInfo, Some(child)) - if (child.children == Nil && !child.isInstanceOf[BackReference] && child.dataType == srcDataType) => - FieldExpression(fNm, fInfo, None) - case fe@FieldExpression(fNm, fInfo, Some(child)) if isSrc(child) => - throw new ExpressionException(fe, s"srcType of field doesn't match input type") - case hasFieldUnaryExpression(fNm, child) if child.dataType == srcDataType => - hasFieldLeafExpression(fNm, Some(child)) - case hF@hasFieldUnaryExpression(fNm, child) if isSrc(child) => - throw new ExpressionException(hF, s"srcType of field doesn't match input type") - case isTraitUnaryExpression(fNm, child) if child.dataType == srcDataType => - isTraitLeafExpression(fNm) - case iT@isTraitUnaryExpression(fNm, child) if isSrc(child) => - throw new ExpressionException(iT, s"srcType of field doesn't match input type") - } - - def validateOnlyFieldReferencesInLoopExpressions(loopExpr: LoopExpression) - : PartialFunction[Expression, Unit] = { - case f: FieldExpression => () - case x => throw new ExpressionException(loopExpr, - s"Loop Expression can only contain field references; '${x.toString}' not supported.") - } - - def apply(e: Expression): Expression = e match { - case f@FilterExpression(inputExpr, condExpr) => { - val validatedCE = condExpr.transformUp(validateQualifiedField(inputExpr.dataType)) - if (validatedCE.fastEquals(condExpr)) { - f - } else { - new FilterExpression(inputExpr, validatedCE) - } - } - case SelectExpression(child, selectList, forGroupBy) if child.resolved => { - val v = validateQualifiedField(child.dataType) - return new SelectExpression(child, selectList.map { - _.transformUp(v) - }, forGroupBy) - } - case OrderExpression(child, order, asc) => { - val v = validateQualifiedField(child.dataType) - OrderExpression(child, order.transformUp(v), asc) - } - case l@LoopExpression(inputExpr, loopExpr, t) => { - val validatedLE = loopExpr.transformUp(validateQualifiedField(inputExpr.dataType)) - val l1 = { - if (validatedLE.fastEquals(loopExpr)) l - else new LoopExpression(inputExpr, validatedLE, t) - } - l1.loopingExpression.traverseUp(validateOnlyFieldReferencesInLoopExpressions(l1)) - l1 - } - case x => x - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala ---------------------------------------------------------------------- diff --git a/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala b/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala deleted file mode 100755 index 8d2c7ae..0000000 --- a/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala +++ /dev/null @@ -1,279 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.query - -import java.util -import java.util.concurrent.atomic.AtomicInteger - -import org.apache.atlas.AtlasException -import org.apache.atlas.query.Expressions.{LimitExpression, PathExpression, SelectExpression} -import org.apache.atlas.repository.Constants -import org.apache.atlas.repository.graph.GraphHelper -import org.apache.atlas.typesystem.types.DataTypes.{ArrayType, PrimitiveType, TypeCategory} -import org.apache.atlas.typesystem.types._ - -object TypeUtils { - val typSystem = TypeSystem.getInstance() - - def numericTypes : Seq[PrimitiveType[_]] = Seq(DataTypes.BYTE_TYPE, - DataTypes.SHORT_TYPE, - DataTypes.INT_TYPE, - DataTypes.FLOAT_TYPE, - DataTypes.LONG_TYPE, - DataTypes.DOUBLE_TYPE, - DataTypes.BIGINTEGER_TYPE, - DataTypes.BIGDECIMAL_TYPE) - - def combinedType(typ1 : IDataType[_], typ2 : IDataType[_]) : PrimitiveType[_] = { - val typ1Idx = if (numericTypes.contains(typ1)) Some(numericTypes.indexOf(typ1)) else None - val typ2Idx = if (numericTypes.contains(typ2)) Some(numericTypes.indexOf(typ2)) else None - - if ( typ1Idx.isDefined && typ2Idx.isDefined ) { - val rIdx = math.max(typ1Idx.get, typ2Idx.get) - - if ( (typ1 == DataTypes.FLOAT_TYPE && typ2 == DataTypes.LONG_TYPE) || - (typ1 == DataTypes.LONG_TYPE && typ2 == DataTypes.FLOAT_TYPE) ) { - return DataTypes.DOUBLE_TYPE - } - return numericTypes(rIdx) - } - - throw new AtlasException(s"Cannot combine types: ${typ1.getName} and ${typ2.getName}") - } - - var tempStructCounter : AtomicInteger = new AtomicInteger(0) - val TEMP_STRUCT_NAME_PREFIX = "__tempQueryResultStruct" - def createStructType(selectExprs : List[Expressions.AliasExpression]) : StructType = { - val aDefs = new Array[AttributeDefinition](selectExprs.size) - selectExprs.zipWithIndex.foreach { t => - val (e,i) = t - aDefs(i) = new AttributeDefinition(e.alias,e.dataType.getName, Multiplicity.OPTIONAL, false, null) - } - return typSystem.defineQueryResultType(s"${TEMP_STRUCT_NAME_PREFIX}${tempStructCounter.getAndIncrement}", - null, - aDefs:_*); - } - - object ResultWithPathStruct { - val pathAttrName = "path" - val resultAttrName = "result" - val pathAttrType = DataTypes.arrayTypeName(typSystem.getIdType.getStructType) - - val pathAttr = new AttributeDefinition(pathAttrName, pathAttrType, Multiplicity.COLLECTION, false, null) - - def createType(pE : PathExpression, resultType : IDataType[_]) : StructType = { - val resultAttr = new AttributeDefinition(resultAttrName, resultType.getName, Multiplicity.REQUIRED, false, null) - val typName = s"${TEMP_STRUCT_NAME_PREFIX}${tempStructCounter.getAndIncrement}" - val m : java.util.HashMap[String, IDataType[_]] = new util.HashMap[String, IDataType[_]]() - if (pE.child.isInstanceOf[SelectExpression] || pE.child.isInstanceOf[LimitExpression]) { - m.put(pE.child.dataType.getName, pE.child.dataType) - } - typSystem.defineQueryResultType(typName, m, pathAttr, resultAttr); - } - } - - /** - * Structure representing the Closure Graph. - * Returns: - * 1. A map of vertexId -> vertex Info(these are the attributes requested in the query) - * 2. A edges map: each entry is a mapping from an vertexId to the List of adjacent vertexIds. - * - * '''The Vertex Map doesn't contain all the vertices in the Graph. Only the ones for which Attributes are - * available.''' These are the vertices that represent the EntityType whose Closure was requested. For e.g. for - * Table Lineage the ''vertex map'' will contain information about Tables, but not about ''Load Process'' vertices - * that connect Tables. - */ - object GraphResultStruct { - val SRC_PREFIX = "src" - val DEST_PREFIX = "dest" - - val verticesAttrName = "vertices" - val edgesAttrName = "edges" - val vertexIdAttrName = "vertexId" - - lazy val edgesAttrType = typSystem.defineMapType(DataTypes.STRING_TYPE, - typSystem.defineArrayType(DataTypes.STRING_TYPE)) - - def createType(resultWithPathType: StructType): StructType = { - val resultType = resultWithPathType.fieldMapping().fields.get(ResultWithPathStruct.resultAttrName).dataType() - - val verticesAttrType = typSystem.defineMapType(DataTypes.STRING_TYPE, - vertexType(resultType.asInstanceOf[StructType])) - val typName = s"${TEMP_STRUCT_NAME_PREFIX}${tempStructCounter.getAndIncrement}" - val verticesAttr = new AttributeDefinition(verticesAttrName, verticesAttrType.getName, - Multiplicity.REQUIRED, false, null) - val edgesAttr = new AttributeDefinition(edgesAttrName, edgesAttrType.getName, Multiplicity.REQUIRED, false, null) - - val m: java.util.HashMap[String, IDataType[_]] = new util.HashMap[String, IDataType[_]]() - m.put(resultWithPathType.getName, resultWithPathType) - m.put(resultType.getName, resultType) - m.put(edgesAttrType.getName, edgesAttrType) - m.put(verticesAttrType.getName, verticesAttrType) - typSystem.defineQueryResultType(typName, m, verticesAttr, edgesAttr) - } - - private def vertexType(resultType: StructType): StructType = { - - import scala.collection.JavaConverters._ - - var attrs: List[AttributeDefinition] = - resultType.fieldMapping.fields.asScala.filter(_._1.startsWith(s"${SRC_PREFIX}_")).mapValues { aInfo => - - new AttributeDefinition(aInfo.name.substring(s"${SRC_PREFIX}_".length), aInfo.dataType.getName, - aInfo.multiplicity, aInfo.isComposite, aInfo.reverseAttributeName) - }.values.toList - - attrs = new AttributeDefinition(vertexIdAttrName, typSystem.getIdType.getStructType.name, - Multiplicity.REQUIRED, false, null) :: attrs - - return typSystem.defineQueryResultType(s"${TEMP_STRUCT_NAME_PREFIX}${tempStructCounter.getAndIncrement}", - null, - attrs: _*) - } - } - - def fieldMapping(iDataType: IDataType[_]) : Option[FieldMapping] = iDataType match { - case c : ClassType => Some(c.fieldMapping()) - case t : TraitType => Some(t.fieldMapping()) - case s : StructType => Some(s.fieldMapping()) - case _ => None - } - - def hasFields(iDataType: IDataType[_]) : Boolean = { - fieldMapping(iDataType).isDefined - } - - import scala.language.existentials - case class FieldInfo(dataType : IDataType[_], - attrInfo : AttributeInfo, - reverseDataType : IDataType[_] = null, - traitName : String = null) { - def isReverse = reverseDataType != null - override def toString : String = { - if ( traitName != null ) { - s"""FieldInfo("${dataType.getName}", "$traitName")""" - } - else if ( reverseDataType == null ) { - s"""FieldInfo("${dataType.getName}", "${attrInfo.name}")""" - } else { - s"""FieldInfo("${dataType.getName}", "${attrInfo.name}", "${reverseDataType.getName}")""" - } - } - } - - val FIELD_QUALIFIER = "(.*?)(->.*)?".r - - /** - * Given a ComposedType `t` and a name resolve using the following rules: - * - if `id` is a field in `t` resolve to the field - * - if `id` is the name of a Struct|Class|Trait Type and it has a field that is of type `t` then return that type - * - * For e.g. - * 1. if we have types Table(name : String, cols : List[Column]), Column(name : String) then - * `resolveReference(Table, "cols")` resolves to type Column. So a query can be "Table.cols" - * 2. But if we have Table(name : String), Column(name : String, tbl : Table) then "Table.Column" will resolve - * to type Column - * - * This way the language will support navigation even if the relationship is one-sided. - * - * @param typ - * @param id - * @return - */ - def resolveReference(typ : IDataType[_], id : String) : Option[FieldInfo] = { - - val fMap = fieldMapping(typ) - if ( fMap.isDefined) { - - if (fMap.get.fields.containsKey(id)) { - return Some(FieldInfo(typ,fMap.get.fields.get(id))) - } - - val systemField = GraphHelper.getAttributeInfoForSystemAttributes(id) - if (systemField != null) { - return Some(FieldInfo(systemField.dataType(), systemField)) - } - - try { - val FIELD_QUALIFIER(clsNm, rest) = id - val idTyp = typSystem.getDataType(classOf[IDataType[_]], clsNm) - val idTypFMap = fieldMapping(idTyp) - - if (rest != null ) { - val attrNm = rest.substring(2) - - if (idTypFMap.get.fields.containsKey(attrNm)) { - return Some(FieldInfo(typ,idTypFMap.get.fields.get(attrNm), idTyp)) - } - } - - if (idTypFMap.isDefined) { - import scala.collection.JavaConversions._ - val fields: Seq[AttributeInfo] = idTypFMap.get.fields.values().filter { aInfo => - aInfo.dataType() == typ || - ( aInfo.dataType().getTypeCategory == TypeCategory.ARRAY && - aInfo.dataType().asInstanceOf[ArrayType].getElemType == typ - ) - }.toSeq - if (fields.size == 1) { - return Some(FieldInfo(typ, fields(0), idTyp)) - } - /* - * is there only 1 array field of this type? - * If yes resolve to it. - * @todo: allow user to specify the relationship to follow by further qualifying the type. for e.g. - * field("LoadProcess.inputTables") - */ - val aFields = fields.filter { aInfo => aInfo.dataType().getTypeCategory == TypeCategory.ARRAY} - if (aFields.size == 1) { - return Some(FieldInfo(typ, aFields(0), idTyp)) - } - } - } catch { - case _ : AtlasException => None - } - } - None - } - - def resolveAsDataType(id : String) : Option[IDataType[_]] = { - try { - Some(typSystem.getDataType(id)) - } catch { - case _ : AtlasException => None - } - - } - - def resolveAsClassType(id : String) : Option[ClassType] = { - try { - Some(typSystem.getDataType(classOf[ClassType], id)) - } catch { - case _ : AtlasException => None - } - } - - def resolveAsTraitType(id : String) : Option[TraitType] = { - try { - Some(typSystem.getDataType(classOf[TraitType], id)) - } catch { - case _ : AtlasException => None - } - } -}
