performance tunning on where parser.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/a6acec72 Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/a6acec72 Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/a6acec72 Branch: refs/heads/feature/test_daewon Commit: a6acec72396e23863586e040ca7bca3ea00f57d7 Parents: 6216412 Author: SteamShon <[email protected]> Authored: Tue Dec 22 17:02:33 2015 +0900 Committer: SteamShon <[email protected]> Committed: Tue Dec 22 17:02:33 2015 +0900 ---------------------------------------------------------------------- .../com/kakao/s2graph/core/QueryParam.scala | 8 ++++- .../s2graph/core/parsers/WhereParser.scala | 31 ++++++++++++-------- .../kakao/s2graph/core/rest/RequestParser.scala | 7 +++-- .../storage/hbase/AsynchbaseQueryBuilder.scala | 2 +- 4 files changed, 31 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a6acec72/s2core/src/main/scala/com/kakao/s2graph/core/QueryParam.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/QueryParam.scala b/s2core/src/main/scala/com/kakao/s2graph/core/QueryParam.scala index 0cefb3f..a903f4d 100644 --- a/s2core/src/main/scala/com/kakao/s2graph/core/QueryParam.scala +++ b/s2core/src/main/scala/com/kakao/s2graph/core/QueryParam.scala @@ -258,6 +258,7 @@ case class QueryParam(labelWithDir: LabelWithDirection, timestamp: Long = System var hasFilters: Map[Byte, InnerValLike] = Map.empty[Byte, InnerValLike] var where: Try[Where] = Success(WhereParser.success) + var whereRawOpt: Option[String] = None var duplicatePolicy = DuplicatePolicy.First var rpcTimeoutInMillis = 1000 var maxAttempt = 2 @@ -297,7 +298,7 @@ case class QueryParam(labelWithDir: LabelWithDirection, timestamp: Long = System def toCacheKeyRaw(bytes: Array[Byte]): Array[Byte] = { val transformBytes = transformer.toHashKeyBytes //TODO: change this to binrary format. - val whereBytes = Bytes.toBytes(where.toString()) + val whereBytes = Bytes.toBytes(whereRawOpt.getOrElse("")) val durationBytes = duration.map { case (min, max) => val minTs = min / cacheTTLInMillis val maxTs = max / cacheTTLInMillis @@ -457,6 +458,11 @@ case class QueryParam(labelWithDir: LabelWithDirection, timestamp: Long = System this } + def whereRawOpt(sqlOpt: Option[String]): QueryParam = { + this.whereRawOpt = sqlOpt + this + } + def isSnapshotEdge = tgtVertexInnerIdOpt.isDefined override def toString = { http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a6acec72/s2core/src/main/scala/com/kakao/s2graph/core/parsers/WhereParser.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/parsers/WhereParser.scala b/s2core/src/main/scala/com/kakao/s2graph/core/parsers/WhereParser.scala index db28eee..15996ea 100644 --- a/s2core/src/main/scala/com/kakao/s2graph/core/parsers/WhereParser.scala +++ b/s2core/src/main/scala/com/kakao/s2graph/core/parsers/WhereParser.scala @@ -47,18 +47,22 @@ trait ExtractValue extends JSONParser { } } - private def findParentEdge(edge: Edge, key: String): (String, Edge) = { - @tailrec def find(edge: Edge, depth: Int): Edge = - if (depth > 0) find(edge.parentEdges.head.edge, depth - 1) - else edge + @tailrec + private def findParent(edge: Edge, depth: Int): Edge = + if (depth > 0) findParent(edge.parentEdges.head.edge, depth - 1) + else edge - val split = key.split(parent) - val depth = split.length - 1 - val propKey = split.last + private def findParentEdge(edge: Edge, key: String): (String, Edge) = { + if (!key.startsWith(parent)) (key, edge) + else { + val split = key.split(parent) + val depth = split.length - 1 + val propKey = split.last - val parentEdge = find(edge, depth) + val parentEdge = findParent(edge, depth) - (propKey, parentEdge) + (propKey, parentEdge) + } } } @@ -97,9 +101,12 @@ case class Eq(propKey: String, value: String) extends Clause { case class IN(propKey: String, values: Set[String]) extends Clause { override def filter(edge: Edge): Boolean = { val propVal = propToInnerVal(edge, propKey) - val valuesToCompare = values.map { value => valueToCompare(edge, propKey, value) } - - valuesToCompare.contains(propVal) + values.exists { value => + valueToCompare(edge, propKey, value) == propVal + } +// val valuesToCompare = values.map { value => valueToCompare(edge, propKey, value) } +// +// valuesToCompare.contains(propVal) } } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a6acec72/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala index 5b1fe1b..96d8f60 100644 --- a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala +++ b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala @@ -92,8 +92,8 @@ class RequestParser(config: Config) extends JSONParser { ret.map(_.toMap).getOrElse(Map.empty[Byte, InnerValLike]) } - def extractWhere(labelMap: Map[String, Label], jsValue: JsValue) = { - (jsValue \ "where").asOpt[String] match { + def extractWhere(labelMap: Map[String, Label], whereClauseOpt: Option[String]) = { + whereClauseOpt match { case None => Success(WhereParser.success) case Some(where) => WhereParser(labelMap).parse(where) match { @@ -279,7 +279,8 @@ class RequestParser(config: Config) extends JSONParser { case None => label.indexSeqsMap.get(scoring.map(kv => kv._1)).map(_.seq).getOrElse(LabelIndex.DefaultSeq) case Some(indexName) => label.indexNameMap.get(indexName).map(_.seq).getOrElse(throw new RuntimeException("cannot find index")) } - val where = extractWhere(labelMap, labelGroup) + val whereClauseOpt = (labelGroup \ "where").asOpt[String] + val where = extractWhere(labelMap, whereClauseOpt) val includeDegree = (labelGroup \ "includeDegree").asOpt[Boolean].getOrElse(true) val rpcTimeout = (labelGroup \ "rpcTimeout").asOpt[Int].getOrElse(DefaultRpcTimeout) val maxAttempt = (labelGroup \ "maxAttempt").asOpt[Int].getOrElse(DefaultMaxAttempt) http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a6acec72/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala index 3d4b5c1..d8e4099 100644 --- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala +++ b/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala @@ -27,7 +27,7 @@ class AsynchbaseQueryBuilder(storage: AsynchbaseStorage)(implicit ec: ExecutionC val expreAfterAccess = storage.config.getInt("future.cache.expire.after.access") val futureCache = CacheBuilder.newBuilder() - .recordStats() +// .recordStats() .initialCapacity(maxSize) .concurrencyLevel(Runtime.getRuntime.availableProcessors()) .expireAfterWrite(expreAfterWrite, TimeUnit.MILLISECONDS)
