Repository: incubator-s2graph Updated Branches: refs/heads/master b769d7ce9 -> ae482d5cc
[S2GRAPH-16] Performance tuning on sort by scoreSum on PostProcess Add InWithoutParent on Where parser for performance tunning. JIRA: [S2GRAPH-16] https://issues.apache.org/jira/browse/S2GRAPH-16 Pull Request: Closes #6 Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/12f721dc Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/12f721dc Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/12f721dc Branch: refs/heads/master Commit: 12f721dc4f4d357074419bee2e65027fbe069c11 Parents: b769d7c Author: DO YUNG YOON <[email protected]> Authored: Tue Feb 23 15:14:35 2016 +0900 Committer: DO YUNG YOON <[email protected]> Committed: Tue Feb 23 15:14:35 2016 +0900 ---------------------------------------------------------------------- CHANGES | 2 + .../s2graph/core/parsers/WhereParser.scala | 55 ++++++++++++++------ .../kakao/s2graph/core/rest/RequestParser.scala | 9 ++-- .../s2graph/core/parsers/WhereParserTest.scala | 15 +++--- 4 files changed, 54 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/12f721dc/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index f0d2f8d..afd2485 100644 --- a/CHANGES +++ b/CHANGES @@ -12,6 +12,8 @@ Release 0.12.1 - unreleased S2GRAPH-11: Performance tuning on sort by scoreSum on PostProcess (Committed by DOYUNG YOON). + S2GRAPH-16: Performance tunning on where parser (Committed by DOYUNG YOON). + BUG FIXES TASKS http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/12f721dc/s2core/src/main/scala/com/kakao/s2graph/core/parsers/WhereParser.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/parsers/WhereParser.scala b/s2core/src/main/scala/com/kakao/s2graph/core/parsers/WhereParser.scala index db28eee..2a62b3b 100644 --- a/s2core/src/main/scala/com/kakao/s2graph/core/parsers/WhereParser.scala +++ b/s2core/src/main/scala/com/kakao/s2graph/core/parsers/WhereParser.scala @@ -47,18 +47,22 @@ trait ExtractValue extends JSONParser { } } - private def findParentEdge(edge: Edge, key: String): (String, Edge) = { - @tailrec def find(edge: Edge, depth: Int): Edge = - if (depth > 0) find(edge.parentEdges.head.edge, depth - 1) - else edge + @tailrec + private def findParent(edge: Edge, depth: Int): Edge = + if (depth > 0) findParent(edge.parentEdges.head.edge, depth - 1) + else edge - val split = key.split(parent) - val depth = split.length - 1 - val propKey = split.last + private def findParentEdge(edge: Edge, key: String): (String, Edge) = { + if (!key.startsWith(parent)) (key, edge) + else { + val split = key.split(parent) + val depth = split.length - 1 + val propKey = split.last - val parentEdge = find(edge, depth) + val parentEdge = findParent(edge, depth) - (propKey, parentEdge) + (propKey, parentEdge) + } } } @@ -94,12 +98,28 @@ case class Eq(propKey: String, value: String) extends Clause { override def filter(edge: Edge): Boolean = binaryOp(_ == _)(propKey, value)(edge) } -case class IN(propKey: String, values: Set[String]) extends Clause { +case class InWithoutParent(label: Label, propKey: String, values: Set[String]) extends Clause { + val innerValLikeLs = values.map { value => + val labelMeta = label.metaPropsInvMap.getOrElse(propKey, throw WhereParserException(s"Where clause contains not existing property name: $propKey")) + val dataType = propKey match { + case "_to" | "to" => label.tgtColumn.columnType + case "_from" | "from" => label.srcColumn.columnType + case _ => labelMeta.dataType + } + toInnerVal(value, dataType, label.schemaVersion) + } override def filter(edge: Edge): Boolean = { val propVal = propToInnerVal(edge, propKey) - val valuesToCompare = values.map { value => valueToCompare(edge, propKey, value) } + innerValLikeLs.contains(propVal) + } +} - valuesToCompare.contains(propVal) +case class IN(propKey: String, values: Set[String]) extends Clause { + override def filter(edge: Edge): Boolean = { + val propVal = propToInnerVal(edge, propKey) + values.exists { value => + valueToCompare(edge, propKey, value) == propVal + } } } @@ -129,7 +149,7 @@ object WhereParser { val success = Where() } -case class WhereParser(labelMap: Map[String, Label]) extends JavaTokenParsers with JSONParser { +case class WhereParser(label: Label) extends JavaTokenParsers with JSONParser { val anyStr = "[^\\s(),]+".r @@ -167,8 +187,13 @@ case class WhereParser(labelMap: Map[String, Label]) extends JavaTokenParsers wi case f ~ minV ~ maxV => Between(f, minV, maxV) } | identWithDot ~ (notIn | in) ~ ("(" ~> repsep(anyStr, ",") <~ ")") ^^ { case f ~ op ~ values => - if (op.toLowerCase == "in") IN(f, values.toSet) - else Not(IN(f, values.toSet)) + val inClause = + if (f.startsWith("_parent")) IN(f, values.toSet) + else InWithoutParent(label, f, values.toSet) + if (op.toLowerCase == "in") inClause + else Not(inClause) + + case _ => throw WhereParserException(s"Failed to parse where clause. ") } } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/12f721dc/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala index 78fc375..90f011c 100644 --- a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala +++ b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala @@ -90,11 +90,11 @@ class RequestParser(config: Config) extends JSONParser { ret.map(_.toMap).getOrElse(Map.empty[Byte, InnerValLike]) } - def extractWhere(labelMap: Map[String, Label], jsValue: JsValue) = { - (jsValue \ "where").asOpt[String] match { + def extractWhere(label: Label, whereClauseOpt: Option[String]) = { + whereClauseOpt match { case None => Success(WhereParser.success) case Some(where) => - WhereParser(labelMap).parse(where) match { + WhereParser(label).parse(where) match { case s@Success(_) => s case Failure(ex) => throw BadQueryException(ex.getMessage, ex) } @@ -272,7 +272,8 @@ class RequestParser(config: Config) extends JSONParser { case None => label.indexSeqsMap.get(scoring.map(kv => kv._1)).map(_.seq).getOrElse(LabelIndex.DefaultSeq) case Some(indexName) => label.indexNameMap.get(indexName).map(_.seq).getOrElse(throw new RuntimeException("cannot find index")) } - val where = extractWhere(labelMap, labelGroup) + val whereClauseOpt = (labelGroup \ "where").asOpt[String] + val where = extractWhere(label, whereClauseOpt) val includeDegree = (labelGroup \ "includeDegree").asOpt[Boolean].getOrElse(true) val rpcTimeout = (labelGroup \ "rpcTimeout").asOpt[Int].getOrElse(DefaultRpcTimeout) val maxAttempt = (labelGroup \ "maxAttempt").asOpt[Int].getOrElse(DefaultMaxAttempt) http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/12f721dc/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala index de19a75..208f062 100644 --- a/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala +++ b/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala @@ -11,7 +11,7 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels { // dummy data for dummy edge initTests() - + import HBaseType.{VERSION1, VERSION2} val ts = System.currentTimeMillis() @@ -32,10 +32,9 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels { (srcId, tgtId, srcIdStr, tgtIdStr, srcVertex, tgtVertex, srcVertexStr, tgtVertexStr, version) } - val labelMap = Map(label.label -> label) - def validate(labelMap: Map[String, Label])(edge: Edge)(sql: String)(expected: Boolean) = { - val whereOpt = WhereParser(labelMap).parse(sql) + def validate(label: Label)(edge: Edge)(sql: String)(expected: Boolean) = { + val whereOpt = WhereParser(label).parse(sql) whereOpt.isSuccess shouldBe true println("=================================================================") @@ -60,7 +59,7 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels { val js = Json.obj("is_hidden" -> true, "is_blocked" -> false, "weight" -> 10, "time" -> 3, "name" -> "abc") val propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs val edge = Edge(srcVertex, tgtVertex, labelWithDir, 0.toByte, ts, propsInner) - val f = validate(labelMap)(edge) _ + val f = validate(label)(edge) _ /** labelName label is long-long relation */ f(s"_to=${tgtVertex.innerId.toString}")(true) @@ -80,7 +79,7 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels { val propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs val edge = Edge(srcVertex, tgtVertex, labelWithDir, 0.toByte, ts, propsInner) - val f = validate(labelMap)(edge) _ + val f = validate(label)(edge) _ // time == 3 f("time >= 3")(true) @@ -110,7 +109,7 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels { val labelWithDirection = if (schemaVer == VERSION2) labelWithDirV2 else labelWithDir val edge = Edge(srcVertex, tgtVertex, labelWithDirection, 0.toByte, ts, propsInner) val lname = if (schemaVer == VERSION2) labelNameV2 else labelName - val f = validate(labelMap)(edge) _ + val f = validate(label)(edge) _ f(s"_from = -1 or _to = ${tgtVertex.innerId.value}")(true) f(s"_from = ${srcVertex.innerId.value} and _to = ${tgtVertex.innerId.value}")(true) @@ -142,7 +141,7 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels { println(parentEdge.toString) println(grandParentEdge.toString) - val f = validate(labelMap)(edge) _ + val f = validate(label)(edge) _ // Compare edge's prop(`_from`) with edge's prop(`name`) f("_from = 1")(true)
