Repository: incubator-s2graph Updated Branches: refs/heads/master 5ac470e91 -> ab032836f
[S2GRAPH-41]: Refactor PostProcess's toSimpleVertexArrJson. seperate logic for buildRawEdges and buildResultJsValue on PostProcess. refactor options on query as queryOption. JIRA: [S2GRAPH-41] https://issues.apache.org/jira/browse/S2GRAPH-41 Pull Request: Closes #24 Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/ab032836 Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/ab032836 Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/ab032836 Branch: refs/heads/master Commit: ab032836f2813a2cb88aa3bfc88f7c0662ff5ca9 Parents: 5ac470e Author: DO YUNG YOON <[email protected]> Authored: Tue Feb 23 17:24:36 2016 +0900 Committer: DO YUNG YOON <[email protected]> Committed: Tue Feb 23 17:24:36 2016 +0900 ---------------------------------------------------------------------- CHANGES | 2 + .../com/kakao/s2graph/core/PostProcess.scala | 274 +++++++++++-------- .../com/kakao/s2graph/core/QueryParam.scala | 56 ++-- .../kakao/s2graph/core/rest/RequestParser.scala | 89 +++--- .../com/kakao/s2graph/core/QueryParamTest.scala | 3 - .../s2graph/core/parsers/WhereParserTest.scala | 3 +- 6 files changed, 254 insertions(+), 173 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/ab032836/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 0a0a10b..d4cb091 100644 --- a/CHANGES +++ b/CHANGES @@ -27,6 +27,8 @@ Release 0.12.1 - unreleased S2GRAPH-40: PostProcess.toHashKey gives different hash key with same value when edge's label schema version is different (Committed by DOYUNG YOON). + S2GRAPH-41: Refactor PostProcess's toSimpleVertexArrJson (Committed by DOYUNG YOON). + BUG FIXES S2GRAPH-18: Query Option "interval" is Broken. http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/ab032836/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala b/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala index 4289c87..702ccf0 100644 --- a/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala +++ b/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala @@ -1,12 +1,12 @@ package com.kakao.s2graph.core import com.kakao.s2graph.core.GraphExceptions.BadQueryException - -import com.kakao.s2graph.core.mysqls.{ColumnMeta, Label, ServiceColumn, LabelMeta} +import com.kakao.s2graph.core._ +import com.kakao.s2graph.core.mysqls._ import com.kakao.s2graph.core.types.{InnerVal, InnerValLike} -import com.kakao.s2graph.core.utils.logger import play.api.libs.json.{Json, _} -import scala.collection.mutable.{ArrayBuffer, ListBuffer} + +import scala.collection.mutable.ListBuffer object PostProcess extends JSONParser { @@ -141,15 +141,13 @@ object PostProcess extends JSONParser { toSimpleVertexArrJson(queryRequestWithResultLs, Seq.empty[QueryRequestWithResult]) } - private def orderBy(q: Query, - orderByColumns: Seq[(String, Boolean)], - rawEdges: ListBuffer[(Map[String, JsValue], Double, (Any, Any, Any, Any))]) - : ListBuffer[(Map[String, JsValue], Double, (Any, Any, Any, Any))] = { + private def orderBy(queryOption: QueryOption, + rawEdges: ListBuffer[(EDGE_VALUES, Double, ORDER_BY_VALUES)]): ListBuffer[(EDGE_VALUES, Double, ORDER_BY_VALUES)] = { import com.kakao.s2graph.core.OrderingUtil._ - if (q.withScore && orderByColumns.nonEmpty) { - val ascendingLs = orderByColumns.map(_._2) - rawEdges.sortBy(_._3)(OrderingUtil.TupleMultiOrdering[Any](ascendingLs)) + if (queryOption.withScore && queryOption.orderByColumns.nonEmpty) { + val ascendingLs = queryOption.orderByColumns.map(_._2) + rawEdges.sortBy(_._3)(TupleMultiOrdering[Any](ascendingLs)) } else { rawEdges } @@ -167,129 +165,183 @@ object PostProcess extends JSONParser { } } - def toSimpleVertexArrJson(queryRequestWithResultLs: Seq[QueryRequestWithResult], exclude: Seq[QueryRequestWithResult]): JsValue = { - val excludeIds = resultInnerIds(exclude).map(innerId => innerId -> true).toMap - + private def buildRawEdges(queryOption: QueryOption, + queryRequestWithResultLs: Seq[QueryRequestWithResult], + excludeIds: Map[Int, Boolean], + scoreWeight: Double = 1.0): (ListBuffer[JsValue], ListBuffer[RAW_EDGE]) = { val degrees = ListBuffer[JsValue]() - val rawEdges = ListBuffer[(Map[String, JsValue], Double, (Any, Any, Any, Any))]() - - if (queryRequestWithResultLs.isEmpty) { - Json.obj("size" -> 0, "degrees" -> Json.arr(), "results" -> Json.arr()) - } else { - val (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResultLs.head).get - val query = queryRequest.query - val queryParam = queryRequest.queryParam - - val orderByColumns = query.orderByColumns.filter { case (column, _) => - column match { - case "from" | "to" | "label" | "score" | "timestamp" | "_timestamp" => true - case _ => - queryParam.label.metaPropNames.contains(column) - } + val rawEdges = ListBuffer[(EDGE_VALUES, Double, ORDER_BY_VALUES)]() + val metaPropNamesMap = scala.collection.mutable.Map[String, Int]() + for { + queryRequestWithResult <- queryRequestWithResultLs + } yield { + val (queryRequest, _) = QueryRequestWithResult.unapply(queryRequestWithResult).get + queryRequest.queryParam.label.metaPropNames.foreach { metaPropName => + metaPropNamesMap.put(metaPropName, metaPropNamesMap.getOrElse(metaPropName, 0) + 1) } + } + val propsExistInAll = metaPropNamesMap.filter(kv => kv._2 == queryRequestWithResultLs.length) + val orderByColumns = queryOption.orderByColumns.filter { case (column, _) => + column match { + case "from" | "to" | "label" | "score" | "timestamp" | "_timestamp" => true + case _ => + propsExistInAll.contains(column) +// //TODO?? +// false +// queryParam.label.metaPropNames.contains(column) + } + } - /** build result jsons */ - for { - queryRequestWithResult <- queryRequestWithResultLs - (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get - queryParam = queryRequest.queryParam - edgeWithScore <- queryResult.edgeWithScoreLs - (edge, score) = EdgeWithScore.unapply(edgeWithScore).get - if !excludeIds.contains(toHashKey(edge, queryRequest.queryParam, query.filterOutFields)) - } { - // edge to json - val (srcColumn, _) = queryParam.label.srcTgtColumn(edge.labelWithDir.dir) - val fromOpt = innerValToJsValue(edge.srcVertex.id.innerId, srcColumn.columnType) - if (edge.isDegree && fromOpt.isDefined) { + /** build result jsons */ + for { + queryRequestWithResult <- queryRequestWithResultLs + (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get + queryParam = queryRequest.queryParam + edgeWithScore <- queryResult.edgeWithScoreLs + (edge, score) = EdgeWithScore.unapply(edgeWithScore).get + hashKey = toHashKey(edge, queryRequest.queryParam, queryOption.filterOutFields) + if !excludeIds.contains(hashKey) + } { + + // edge to json + val (srcColumn, _) = queryParam.label.srcTgtColumn(edge.labelWithDir.dir) + val fromOpt = innerValToJsValue(edge.srcVertex.id.innerId, srcColumn.columnType) + if (edge.isDegree && fromOpt.isDefined) { + if (queryOption.limitOpt.isEmpty) { degrees += Json.obj( "from" -> fromOpt.get, "label" -> queryRequest.queryParam.label.label, "direction" -> GraphUtil.fromDirection(edge.labelWithDir.dir), LabelMeta.degree.name -> innerValToJsValue(edge.propsWithTs(LabelMeta.degreeSeq).innerVal, InnerVal.LONG) ) - } else { - val keyWithJs = edgeToJson(edge, score, queryRequest.query, queryRequest.queryParam) - val orderByValues: (Any, Any, Any, Any) = orderByColumns.length match { - case 0 => - (None, None, None, None) - case 1 => - val it = orderByColumns.iterator - val v1 = getColumnValue(keyWithJs, score, edge, it.next()._1) - (v1, None, None, None) - case 2 => - val it = orderByColumns.iterator - val v1 = getColumnValue(keyWithJs, score, edge, it.next()._1) - val v2 = getColumnValue(keyWithJs, score, edge, it.next()._1) - (v1, v2, None, None) - case 3 => - val it = orderByColumns.iterator - val v1 = getColumnValue(keyWithJs, score, edge, it.next()._1) - val v2 = getColumnValue(keyWithJs, score, edge, it.next()._1) - val v3 = getColumnValue(keyWithJs, score, edge, it.next()._1) - (v1, v2, v3, None) - case _ => - val it = orderByColumns.iterator - val v1 = getColumnValue(keyWithJs, score, edge, it.next()._1) - val v2 = getColumnValue(keyWithJs, score, edge, it.next()._1) - val v3 = getColumnValue(keyWithJs, score, edge, it.next()._1) - val v4 = getColumnValue(keyWithJs, score, edge, it.next()._1) - (v1, v2, v3, v4) - } - - val currentEdge = (keyWithJs, score, orderByValues) - rawEdges += currentEdge } + } else { + val keyWithJs = edgeToJson(edge, score, queryRequest.query, queryRequest.queryParam) + val orderByValues: (Any, Any, Any, Any) = orderByColumns.length match { + case 0 => + (None, None, None, None) + case 1 => + val it = orderByColumns.iterator + val v1 = getColumnValue(keyWithJs, score, edge, it.next()._1) + (v1, None, None, None) + case 2 => + val it = orderByColumns.iterator + val v1 = getColumnValue(keyWithJs, score, edge, it.next()._1) + val v2 = getColumnValue(keyWithJs, score, edge, it.next()._1) + (v1, v2, None, None) + case 3 => + val it = orderByColumns.iterator + val v1 = getColumnValue(keyWithJs, score, edge, it.next()._1) + val v2 = getColumnValue(keyWithJs, score, edge, it.next()._1) + val v3 = getColumnValue(keyWithJs, score, edge, it.next()._1) + (v1, v2, v3, None) + case _ => + val it = orderByColumns.iterator + val v1 = getColumnValue(keyWithJs, score, edge, it.next()._1) + val v2 = getColumnValue(keyWithJs, score, edge, it.next()._1) + val v3 = getColumnValue(keyWithJs, score, edge, it.next()._1) + val v4 = getColumnValue(keyWithJs, score, edge, it.next()._1) + (v1, v2, v3, v4) + } + + val currentEdge = (keyWithJs, score * scoreWeight, orderByValues) + rawEdges += currentEdge } + } + (degrees, rawEdges) + } - if (query.groupByColumns.isEmpty) { - // ordering - val edges = orderBy(query, orderByColumns, rawEdges).map(_._1) + private def buildResultJsValue(queryOption: QueryOption, + degrees: ListBuffer[JsValue], + rawEdges: ListBuffer[RAW_EDGE]): JsValue = { + if (queryOption.groupByColumns.isEmpty) { + // ordering + val filteredEdges = rawEdges.filter(t => t._2 >= queryOption.scoreThreshold) - Json.obj( - "size" -> edges.size, - "degrees" -> degrees, - "results" -> edges, - "impressionId" -> query.impressionId() - ) - } else { - val grouped = rawEdges.groupBy { case (keyWithJs, _, _) => - val props = keyWithJs.get("props") - - for { - column <- query.groupByColumns - value <- keyWithJs.get(column) match { - case None => props.flatMap { js => (js \ column).asOpt[JsValue] } - case Some(x) => Some(x) - } - } yield column -> value - } + val edges = queryOption.limitOpt match { + case None => orderBy(queryOption, filteredEdges).map(_._1) + case Some(limit) => orderBy(queryOption, filteredEdges).map(_._1).take(limit) + } + val resultDegrees = if (queryOption.returnDegree) degrees else emptyDegrees + + Json.obj( + "size" -> edges.size, + "degrees" -> resultDegrees, + "results" -> edges + ) + } else { + val grouped = rawEdges.groupBy { case (keyWithJs, _, _) => + val props = keyWithJs.get("props") + + for { + column <- queryOption.groupByColumns + value <- keyWithJs.get(column) match { + case None => props.flatMap { js => (js \ column).asOpt[JsValue] } + case Some(x) => Some(x) + } + } yield column -> value + } - val groupedEdgesWithScoreSum = { - for { - (groupByKeyVals, groupedRawEdges) <- grouped - } yield { - val scoreSum = groupedRawEdges.map(x => x._2).sum - // ordering - val edges = orderBy(query, orderByColumns, groupedRawEdges).map(_._1) - val js = Json.obj( + val groupedEdgesWithScoreSum = + for { + (groupByKeyVals, groupedRawEdges) <- grouped + scoreSum = groupedRawEdges.map(x => x._2).sum if scoreSum >= queryOption.scoreThreshold + } yield { + // ordering + val edges = orderBy(queryOption, groupedRawEdges).map(_._1) + + //TODO: refactor this + val js = if (queryOption.returnAgg) + Json.obj( "groupBy" -> Json.toJson(groupByKeyVals.toMap), "scoreSum" -> scoreSum, "agg" -> edges ) - (js, scoreSum) - } + else + Json.obj( + "groupBy" -> Json.toJson(groupByKeyVals.toMap), + "scoreSum" -> scoreSum, + "agg" -> Json.arr() + ) + (js, scoreSum) } - val groupedSortedJsons = groupedEdgesWithScoreSum.toList.sortBy { case (jsVal, scoreSum) => scoreSum * -1 }.map(_._1) - Json.obj( - "size" -> groupedSortedJsons.size, - "degrees" -> degrees, - "results" -> groupedSortedJsons, - "impressionId" -> query.impressionId() - ) + val groupedSortedJsons = queryOption.limitOpt match { + case None => + groupedEdgesWithScoreSum.toList.sortBy { case (jsVal, scoreSum) => scoreSum * -1 }.map(_._1) + case Some(limit) => + groupedEdgesWithScoreSum.toList.sortBy { case (jsVal, scoreSum) => scoreSum * -1 }.map(_._1).take(limit) } + val resultDegrees = if (queryOption.returnDegree) degrees else emptyDegrees + Json.obj( + "size" -> groupedSortedJsons.size, + "degrees" -> resultDegrees, + "results" -> groupedSortedJsons + ) } } + + def toSimpleVertexArrJson(queryOption: QueryOption, + queryRequestWithResultLs: Seq[QueryRequestWithResult], + exclude: Seq[QueryRequestWithResult]): JsValue = { + val excludeIds = resultInnerIds(exclude).map(innerId => innerId -> true).toMap + val (degrees, rawEdges) = buildRawEdges(queryOption, queryRequestWithResultLs, excludeIds) + buildResultJsValue(queryOption, degrees, rawEdges) + } + + + def toSimpleVertexArrJson(queryRequestWithResultLs: Seq[QueryRequestWithResult], + exclude: Seq[QueryRequestWithResult]): JsValue = { + + queryRequestWithResultLs.headOption.map { queryRequestWithResult => + val (queryRequest, _) = QueryRequestWithResult.unapply(queryRequestWithResult).get + val query = queryRequest.query + val queryOption = query.queryOption + val excludeIds = resultInnerIds(exclude).map(innerId => innerId -> true).toMap + val (degrees, rawEdges) = buildRawEdges(queryOption, queryRequestWithResultLs, excludeIds) + buildResultJsValue(queryOption, degrees, rawEdges) + } getOrElse emptyResults + } def verticesToJson(vertices: Iterable[Vertex]) = { Json.toJson(vertices.flatMap { v => vertexToJson(v) }) http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/ab032836/s2core/src/main/scala/com/kakao/s2graph/core/QueryParam.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/QueryParam.scala b/s2core/src/main/scala/com/kakao/s2graph/core/QueryParam.scala index c3a6f17..b0f564f 100644 --- a/s2core/src/main/scala/com/kakao/s2graph/core/QueryParam.scala +++ b/s2core/src/main/scala/com/kakao/s2graph/core/QueryParam.scala @@ -30,31 +30,49 @@ object Query { } } } - } +case class QueryOption(removeCycle: Boolean = false, + selectColumns: Seq[String] = Seq.empty, + groupByColumns: Seq[String] = Seq.empty, + orderByColumns: Seq[(String, Boolean)] = Seq.empty, + filterOutQuery: Option[Query] = None, + filterOutFields: Seq[String] = Seq(LabelMeta.to.name), + withScore: Boolean = true, + returnTree: Boolean = false, + limitOpt: Option[Int] = None, + returnAgg: Boolean = true, + scoreThreshold: Double = Double.MinValue, + returnDegree: Boolean = true) + + case class Query(vertices: Seq[Vertex] = Seq.empty[Vertex], steps: IndexedSeq[Step] = Vector.empty[Step], - removeCycle: Boolean = false, - selectColumns: Seq[String] = Seq.empty[String], - groupByColumns: Seq[String] = Seq.empty[String], - orderByColumns: Seq[(String, Boolean)] = Nil, - filterOutQuery: Option[Query] = None, - filterOutFields: Seq[String] = Seq(LabelMeta.to.name), - withScore: Boolean = true, - returnTree: Boolean = false) { + queryOption: QueryOption = QueryOption()) { + + val removeCycle = queryOption.removeCycle + val selectColumns = queryOption.selectColumns + val groupByColumns = queryOption.groupByColumns + val orderByColumns = queryOption.orderByColumns + val filterOutQuery = queryOption.filterOutQuery + val filterOutFields = queryOption.filterOutFields + val withScore = queryOption.withScore + val returnTree = queryOption.returnTree + val limitOpt = queryOption.limitOpt + val returnAgg = queryOption.returnAgg + val returnDegree = queryOption.returnDegree def cacheKeyBytes: Array[Byte] = { - val selectBytes = Bytes.toBytes(selectColumns.toString) - val groupBytes = Bytes.toBytes(groupByColumns.toString) - val orderByBytes = Bytes.toBytes(orderByColumns.toString) - val filterOutBytes = filterOutQuery.map(_.cacheKeyBytes).getOrElse(Array.empty[Byte]) - val returnTreeBytes = Bytes.toBytes(returnTree) + val selectBytes = Bytes.toBytes(queryOption.selectColumns.toString) + val groupBytes = Bytes.toBytes(queryOption.groupByColumns.toString) + val orderByBytes = Bytes.toBytes(queryOption.orderByColumns.toString) + val filterOutBytes = queryOption.filterOutQuery.map(_.cacheKeyBytes).getOrElse(Array.empty[Byte]) + val returnTreeBytes = Bytes.toBytes(queryOption.returnTree) Seq(selectBytes, groupBytes, orderByBytes, filterOutBytes, returnTreeBytes).foldLeft(Array.empty[Byte])(Bytes.add) } - lazy val selectColumnsSet = selectColumns.map { c => + lazy val selectColumnsSet = queryOption.selectColumns.map { c => if (c == "_from") "from" else if (c == "_to") "to" else c @@ -256,6 +274,7 @@ case class QueryParam(labelWithDir: LabelWithDirection, timestamp: Long = System var hasFilters: Map[Byte, InnerValLike] = Map.empty[Byte, InnerValLike] var where: Try[Where] = Success(WhereParser.success) + var whereRawOpt: Option[String] = None var duplicatePolicy = DuplicatePolicy.First var rpcTimeoutInMillis = 1000 var maxAttempt = 2 @@ -296,7 +315,7 @@ case class QueryParam(labelWithDir: LabelWithDirection, timestamp: Long = System def toCacheKeyRaw(bytes: Array[Byte]): Array[Byte] = { val transformBytes = transformer.toHashKeyBytes //TODO: change this to binrary format. - val whereBytes = Bytes.toBytes(where.toString()) + val whereBytes = Bytes.toBytes(whereRawOpt.getOrElse("")) val durationBytes = duration.map { case (min, max) => val minTs = min / cacheTTLInMillis val maxTs = max / cacheTTLInMillis @@ -456,6 +475,11 @@ case class QueryParam(labelWithDir: LabelWithDirection, timestamp: Long = System this } + def whereRawOpt(sqlOpt: Option[String]): QueryParam = { + this.whereRawOpt = sqlOpt + this + } + def shouldNormalize(shouldNormalize: Boolean): QueryParam = { this.shouldNormalize = shouldNormalize this http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/ab032836/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala index 63c41e0..a14a3b0 100644 --- a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala +++ b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala @@ -5,8 +5,8 @@ import com.kakao.s2graph.core._ import com.kakao.s2graph.core.mysqls._ import com.kakao.s2graph.core.parsers.WhereParser import com.kakao.s2graph.core.types._ +import com.kakao.s2graph.core.utils.logger import com.typesafe.config.Config - import play.api.libs.json._ import scala.util.{Failure, Success, Try} @@ -112,7 +112,48 @@ class RequestParser(config: Config) extends JSONParser { } vertices.toSeq } - + def toQueryOption(jsValue: JsValue): QueryOption = { + val filterOutFields = (jsValue \ "filterOutFields").asOpt[List[String]].getOrElse(List(LabelMeta.to.name)) + val filterOutQuery = (jsValue \ "filterOut").asOpt[JsValue].map { v => toQuery(v) }.map { q => + q.copy(queryOption = q.queryOption.copy(filterOutFields = filterOutFields)) + } + val removeCycle = (jsValue \ "removeCycle").asOpt[Boolean].getOrElse(true) + val selectColumns = (jsValue \ "select").asOpt[List[String]].getOrElse(List.empty) + val groupByColumns = (jsValue \ "groupBy").asOpt[List[String]].getOrElse(List.empty) + val orderByColumns: List[(String, Boolean)] = (jsValue \ "orderBy").asOpt[List[JsObject]].map { jsLs => + for { + js <- jsLs + (column, orderJs) <- js.fields + } yield { + val ascending = orderJs.as[String].toUpperCase match { + case "ASC" => true + case "DESC" => false + } + column -> ascending + } + }.getOrElse(List("score" -> false, "timestamp" -> false)) + val withScore = (jsValue \ "withScore").asOpt[Boolean].getOrElse(true) + val returnTree = (jsValue \ "returnTree").asOpt[Boolean].getOrElse(false) + //TODO: Refactor this + val limitOpt = (jsValue \ "limit").asOpt[Int] + val returnAgg = (jsValue \ "returnAgg").asOpt[Boolean].getOrElse(true) + val scoreThreshold = (jsValue \ "scoreThreshold").asOpt[Double].getOrElse(Double.MinValue) + val returnDegree = (jsValue \ "returnDegree").asOpt[Boolean].getOrElse(true) + + QueryOption(removeCycle = removeCycle, + selectColumns = selectColumns, + groupByColumns = groupByColumns, + orderByColumns = orderByColumns, + filterOutQuery = filterOutQuery, + filterOutFields = filterOutFields, + withScore = withScore, + returnTree = returnTree, + limitOpt = limitOpt, + returnAgg = returnAgg, + scoreThreshold = scoreThreshold, + returnDegree = returnDegree + ) + } def toQuery(jsValue: JsValue, isEdgeQuery: Boolean = true): Query = { try { val vertices = @@ -135,34 +176,9 @@ class RequestParser(config: Config) extends JSONParser { }).flatten if (vertices.isEmpty) throw BadQueryException("srcVertices`s id is empty") - - val filterOutFields = (jsValue \ "filterOutFields").asOpt[List[String]].getOrElse(List(LabelMeta.to.name)) - val filterOutQuery = (jsValue \ "filterOut").asOpt[JsValue].map { v => toQuery(v) }.map { q => q.copy(filterOutFields = filterOutFields) } val steps = parse[Vector[JsValue]](jsValue, "steps") - val removeCycle = (jsValue \ "removeCycle").asOpt[Boolean].getOrElse(true) - val selectColumns = (jsValue \ "select").asOpt[List[String]].getOrElse(List.empty) - val groupByColumns = (jsValue \ "groupBy").asOpt[List[String]].getOrElse(List.empty) - val orderByColumns: List[(String, Boolean)] = (jsValue \ "orderBy").asOpt[List[JsObject]].map { jsLs => - for { - js <- jsLs - (column, orderJs) <- js.fields - } yield { - val ascending = orderJs.as[String].toUpperCase match { - case "ASC" => true - case "DESC" => false - } - column -> ascending - } - }.getOrElse(List("score" -> false, "timestamp" -> false)) - val withScore = (jsValue \ "withScore").asOpt[Boolean].getOrElse(true) - val returnTree = (jsValue \ "returnTree").asOpt[Boolean].getOrElse(false) - // TODO: throw exception, when label dosn't exist - val labelMap = (for { - js <- jsValue \\ "label" - labelName <- js.asOpt[String] - label <- Label.findByName(labelName) - } yield (labelName, label)).toMap + val queryOption = toQueryOption(jsValue) val querySteps = steps.zipWithIndex.map { case (step, stepIdx) => @@ -197,7 +213,7 @@ class RequestParser(config: Config) extends JSONParser { val queryParams = for { labelGroup <- queryParamJsVals - queryParam <- parseQueryParam(labelMap, labelGroup) + queryParam <- parseQueryParam(labelGroup) } yield { val (_, columnName) = if (queryParam.labelWithDir.dir == GraphUtil.directions("out")) { @@ -220,18 +236,7 @@ class RequestParser(config: Config) extends JSONParser { } - val ret = Query( - vertices, - querySteps, - removeCycle = removeCycle, - selectColumns = selectColumns, - groupByColumns = groupByColumns, - orderByColumns = orderByColumns, - filterOutQuery = filterOutQuery, - filterOutFields = filterOutFields, - withScore = withScore, - returnTree = returnTree - ) + val ret = Query(vertices, querySteps, queryOption) // logger.debug(ret.toString) ret } catch { @@ -244,7 +249,7 @@ class RequestParser(config: Config) extends JSONParser { } } - private def parseQueryParam(labelMap: Map[String, Label], labelGroup: JsValue): Option[QueryParam] = { + private def parseQueryParam(labelGroup: JsValue): Option[QueryParam] = { for { labelName <- parse[Option[String]](labelGroup, "label") } yield { http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/ab032836/s2core/src/test/scala/com/kakao/s2graph/core/QueryParamTest.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/QueryParamTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/QueryParamTest.scala index 374293e..544c17c 100644 --- a/s2core/src/test/scala/com/kakao/s2graph/core/QueryParamTest.scala +++ b/s2core/src/test/scala/com/kakao/s2graph/core/QueryParamTest.scala @@ -4,9 +4,6 @@ import com.kakao.s2graph.core.types.LabelWithDirection import org.apache.hadoop.hbase.util.Bytes import org.scalatest.{FunSuite, Matchers} -/** - * Created by shon on 7/31/15. - */ class QueryParamTest extends FunSuite with Matchers with TestCommon { // val version = HBaseType.VERSION2 // val testEdge = Management.toEdge(ts, "insert", "1", "10", labelNameV2, "out", Json.obj("is_blocked" -> true, "phone_number" -> "xxxx", "age" -> 20).toString) http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/ab032836/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala index 208f062..b431cc2 100644 --- a/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala +++ b/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala @@ -1,8 +1,9 @@ package com.kakao.s2graph.core.parsers import com.kakao.s2graph.core._ -import com.kakao.s2graph.core.mysqls.{Label, LabelMeta} +import com.kakao.s2graph.core.mysqls.{Experiment, Label, LabelMeta} import com.kakao.s2graph.core.types._ +import com.kakao.s2graph.core.utils.logger import org.scalatest.{FunSuite, Matchers} import play.api.libs.json.Json
