add test case for search vertex query
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/8ed6d20f Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/8ed6d20f Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/8ed6d20f Branch: refs/heads/master Commit: 8ed6d20faa907e08f43a22360bcbeaab43d273e8 Parents: f187040 Author: daewon <[email protected]> Authored: Fri Apr 20 14:50:17 2018 +0900 Committer: daewon <[email protected]> Committed: Fri Apr 20 14:50:17 2018 +0900 ---------------------------------------------------------------------- .../scala/org/apache/s2graph/core/S2Graph.scala | 1 + .../s2graph/core/index/IndexProvider.scala | 2 - .../core/index/LuceneIndexProvider.scala | 25 ++-- .../graphql/repository/GraphRepository.scala | 7 +- .../s2graph/graphql/types/FieldResolver.scala | 23 +++- .../apache/s2graph/graphql/types/S2Type.scala | 16 +-- .../apache/s2graph/graphql/ScenarioTest.scala | 134 +++++++++++++------ 7 files changed, 130 insertions(+), 78 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/8ed6d20f/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala index 4a28f0a..7f19cb4 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala @@ -476,6 +476,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap val futures = edgesWithIdx.groupBy { case (e, idx) => e.innerLabel }.map { case (label, edgeGroup) => getStorage(label).incrementCounts(label.hbaseZkAddr, edgeGroup.map(_._1), withWait).map(_.zip(edgeGroup.map(_._2))) } + Future.sequence(futures).map { ls => ls.flatten.toSeq.sortBy(_._2).map(_._1) } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/8ed6d20f/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala index ffbebf4..f573ff6 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala @@ -35,8 +35,6 @@ import scala.util.Try object IndexProvider { import GlobalIndex._ - //TODO: Fix Me - val hitsPerPage = 100000 val IdField = "id" def apply(config: Config)(implicit ec: ExecutionContext): IndexProvider = { http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/8ed6d20f/s2core/src/main/scala/org/apache/s2graph/core/index/LuceneIndexProvider.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/index/LuceneIndexProvider.scala b/s2core/src/main/scala/org/apache/s2graph/core/index/LuceneIndexProvider.scala index 8d5f997..16c094a 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/index/LuceneIndexProvider.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/index/LuceneIndexProvider.scala @@ -24,12 +24,12 @@ import java.util import com.typesafe.config.Config import org.apache.lucene.analysis.core.KeywordAnalyzer -import org.apache.lucene.analysis.standard.StandardAnalyzer import org.apache.lucene.document.{Document, Field, StringField} import org.apache.lucene.index.{DirectoryReader, IndexWriter, IndexWriterConfig, Term} import org.apache.lucene.queryparser.classic.{ParseException, QueryParser} import org.apache.lucene.search.{IndexSearcher, Query} import org.apache.lucene.store.{BaseDirectory, RAMDirectory, SimpleFSDirectory} +import org.apache.lucene.search.TopScoreDocCollector import org.apache.s2graph.core.io.Conversions import org.apache.s2graph.core.mysqls.GlobalIndex import org.apache.s2graph.core.types.VertexId @@ -53,6 +53,7 @@ class LuceneIndexProvider(config: Config) extends IndexProvider { val writers = mutable.Map.empty[String, IndexWriter] val directories = mutable.Map.empty[String, BaseDirectory] val baseDirectory = scala.util.Try(config.getString("index.provider.base.dir")).getOrElse(".") + val MAX_RESULTS = 100000 private def getOrElseDirectory(indexName: String): BaseDirectory = { val pathname = s"${baseDirectory}/${indexName}" @@ -136,7 +137,7 @@ class LuceneIndexProvider(config: Config) extends IndexProvider { vertices.foreach { vertex => toDocument(vertex, forceToIndex).foreach { doc => val vId = vertex.id.toString() -// logger.error(s"DOC: ${doc}") + // logger.error(s"DOC: ${doc}") writer.updateDocument(new Term(vidField, vId), doc) } @@ -166,21 +167,21 @@ class LuceneIndexProvider(config: Config) extends IndexProvider { edges.map(_ => true) } - private def fetchInner[T](q: Query, indexKey: String, field: String, reads: Reads[T]): util.List[T] = { + private def fetchInner[T](q: Query, offset: Int, limit: Int, indexKey: String, field: String, reads: Reads[T]): util.List[T] = { val ids = new java.util.HashSet[T] - var reader: DirectoryReader = null + var reader: DirectoryReader = null try { val reader = DirectoryReader.open(getOrElseDirectory(indexKey)) val searcher = new IndexSearcher(reader) + val collector = TopScoreDocCollector.create(MAX_RESULTS) + val startIndex = offset * limit - val docs = searcher.search(q, hitsPerPage) -// logger.error(s"total hit: ${docs.scoreDocs.length}") + searcher.search(q, collector) - docs.scoreDocs.foreach { scoreDoc => + val hits = collector.topDocs(startIndex, limit) + hits.scoreDocs.foreach { scoreDoc => val document = searcher.doc(scoreDoc.doc) -// logger.error(s"DOC_IN_L: ${document.toString}") - val id = reads.reads(Json.parse(document.get(field))).get ids.add(id) } @@ -199,7 +200,7 @@ class LuceneIndexProvider(config: Config) extends IndexProvider { try { val q = new QueryParser(field, analyzer).parse(queryString) - fetchInner[VertexId](q, GlobalIndex.VertexIndexName, vidField, Conversions.s2VertexIdReads) + fetchInner[VertexId](q, 0, 100, GlobalIndex.VertexIndexName, vidField, Conversions.s2VertexIdReads) } catch { case ex: ParseException => logger.error(s"[IndexProvider]: ${queryString} parse failed.", ex) @@ -213,7 +214,7 @@ class LuceneIndexProvider(config: Config) extends IndexProvider { try { val q = new QueryParser(field, analyzer).parse(queryString) - fetchInner[EdgeId](q, GlobalIndex.EdgeIndexName, field, Conversions.s2EdgeIdReads) + fetchInner[EdgeId](q, 0, 100, GlobalIndex.EdgeIndexName, field, Conversions.s2EdgeIdReads) } catch { case ex: ParseException => logger.error(s"[IndexProvider]: ${queryString} parse failed.", ex) @@ -230,7 +231,7 @@ class LuceneIndexProvider(config: Config) extends IndexProvider { val field = vidField try { val q = new QueryParser(field, analyzer).parse(queryString) - fetchInner[VertexId](q, GlobalIndex.VertexIndexName, vidField, Conversions.s2VertexIdReads) + fetchInner[VertexId](q, vertexQueryParam.offset, vertexQueryParam.limit, GlobalIndex.VertexIndexName, vidField, Conversions.s2VertexIdReads) } catch { case ex: ParseException => logger.error(s"[IndexProvider]: ${queryString} parse failed.", ex) http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/8ed6d20f/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala ---------------------------------------------------------------------- diff --git a/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala b/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala index a83b7f2..e5a04b1 100644 --- a/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala +++ b/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala @@ -41,15 +41,14 @@ object GraphRepository { } implicit val edgeHasId = new HasId[(S2VertexLike, QueryParam, Seq[S2EdgeLike]), DeferFetchEdges] { - override def id(value: (S2VertexLike, QueryParam, Seq[S2EdgeLike])): DeferFetchEdges = - DeferFetchEdges(value._1, value._2) + override def id(value: (S2VertexLike, QueryParam, Seq[S2EdgeLike])): DeferFetchEdges = DeferFetchEdges(value._1, value._2) } val vertexFetcher = - Fetcher((ctx: GraphRepository, ids: Seq[VertexQueryParam]) => { + Fetcher((ctx: GraphRepository, queryParams: Seq[VertexQueryParam]) => { implicit val ec = ctx.ec - Future.traverse(ids)(ctx.getVertices).map(vs => ids.zip(vs)) + Future.traverse(queryParams)(ctx.getVertices).map(vs => queryParams.zip(vs)) }) val edgeFetcher = Fetcher((ctx: GraphRepository, ids: Seq[DeferFetchEdges]) => { http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/8ed6d20f/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/FieldResolver.scala ---------------------------------------------------------------------- diff --git a/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/FieldResolver.scala b/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/FieldResolver.scala index 4f092dd..1625f58 100644 --- a/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/FieldResolver.scala +++ b/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/FieldResolver.scala @@ -49,19 +49,29 @@ object FieldResolver { (vertex, qp) } - def serviceColumnOnService(column: ServiceColumn, c: Context[GraphRepository, Any]): (Seq[S2VertexLike], Boolean) = { + def serviceColumnOnService(column: ServiceColumn, c: Context[GraphRepository, Any]): VertexQueryParam = { + val prefix = s"${GlobalIndex.serviceField}:${column.service.serviceName} AND ${GlobalIndex.serviceColumnField}:${column.columnName}" + val ids = c.argOpt[Any]("id").toSeq ++ c.argOpt[List[Any]]("ids").toList.flatten + val offset = c.arg[Int]("offset") + val limit = c.arg[Int]("limit") + val vertices = ids.map(vid => c.ctx.toS2VertexLike(vid, column)) + val searchOpt = c.argOpt[String]("search").map { qs => + if (qs.trim.nonEmpty) s"(${prefix}) AND (${qs})" + else prefix + } val columnFields = column.metasInvMap.keySet val selectedFields = AstHelper.selectedFields(c.astFields) - val canSkipFetch = selectedFields.forall(f => f == "id" || !columnFields(f)) - (vertices, canSkipFetch) + val vertexQueryParam = VertexQueryParam(offset, limit, searchOpt, vertices.map(_.id), !canSkipFetch) + + vertexQueryParam } - def serviceColumnOnLabel(c: Context[GraphRepository, Any]): (S2VertexLike, Boolean) = { + def serviceColumnOnLabel(c: Context[GraphRepository, Any]): VertexQueryParam = { val edge = c.value.asInstanceOf[S2EdgeLike] val vertex = edge.tgtForVertex @@ -69,9 +79,10 @@ object FieldResolver { val selectedFields = AstHelper.selectedFields(c.astFields) val columnFields = column.metasInvMap.keySet - val canSkipFetch = selectedFields.forall(f => f == "id" || !columnFields(f)) - (vertex, canSkipFetch) + val vertexQueryParam = VertexQueryParam(0, 1, None, Seq(vertex.id), !canSkipFetch) + + vertexQueryParam } } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/8ed6d20f/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/S2Type.scala ---------------------------------------------------------------------- diff --git a/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/S2Type.scala b/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/S2Type.scala index b532263..c189fe3 100644 --- a/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/S2Type.scala +++ b/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/S2Type.scala @@ -139,20 +139,15 @@ object S2Type { arguments = List( Argument("id", OptionInputType(toScalarType(column.columnType))), Argument("ids", OptionInputType(ListInputType(toScalarType(column.columnType)))), - Argument("search", OptionInputType(StringType)) + Argument("search", OptionInputType(StringType)), + Argument("offset", OptionInputType(IntType), defaultValue = 0), + Argument("limit", OptionInputType(IntType), defaultValue = 100) ), description = Option("desc here"), resolve = c => { implicit val ec = c.ctx.ec - val (vertices, canSkipFetchVertex) = FieldResolver.serviceColumnOnService(column, c) - val searchOpt = c.argOpt[String]("search").map { qs => - val prefix = s"(${GlobalIndex.serviceField}:${service.serviceName} AND ${GlobalIndex.serviceColumnField}:${column.columnName})" - if (qs.trim.nonEmpty) Seq(prefix, qs).mkString(" AND ") - else prefix - } - - val vertexQueryParam = VertexQueryParam(0, 100, searchOpt, vertices.map(_.id), !canSkipFetchVertex) + val vertexQueryParam = FieldResolver.serviceColumnOnService(column, c) DeferredValue(GraphRepository.vertexFetcher.defer(vertexQueryParam)).map(m => m._2) } ): Field[GraphRepository, Any] @@ -178,9 +173,8 @@ object S2Type { lazy val serviceColumnField: Field[GraphRepository, Any] = Field(column.columnName, labelColumnType, resolve = c => { implicit val ec = c.ctx.ec - val (vertex, canSkipFetchVertex) = FieldResolver.serviceColumnOnLabel(c) + val vertexQueryParam = FieldResolver.serviceColumnOnLabel(c) - val vertexQueryParam = VertexQueryParam(0, 100, None, Seq(vertex.id), !canSkipFetchVertex) DeferredValue(GraphRepository.vertexFetcher.defer(vertexQueryParam)).map(m => m._2.head) }) http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/8ed6d20f/s2graphql/src/test/scala/org/apache/s2graph/graphql/ScenarioTest.scala ---------------------------------------------------------------------- diff --git a/s2graphql/src/test/scala/org/apache/s2graph/graphql/ScenarioTest.scala b/s2graphql/src/test/scala/org/apache/s2graph/graphql/ScenarioTest.scala index 81bf884..58e473d 100644 --- a/s2graphql/src/test/scala/org/apache/s2graph/graphql/ScenarioTest.scala +++ b/s2graphql/src/test/scala/org/apache/s2graph/graphql/ScenarioTest.scala @@ -24,6 +24,7 @@ import org.scalatest._ import play.api.libs.json._ import sangria.execution.ValidationError import sangria.macros._ +import sangria.parser.QueryParser class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll { var testGraph: TestGraph = _ @@ -358,6 +359,11 @@ class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll { id: "shon" age: 19 gender: "F" + }, + { + id: "rain" + age: 21 + gender: "T" }] }) { isSuccess @@ -374,8 +380,10 @@ class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll { "isSuccess": true }, { "isSuccess": true - }] - } + }, { + "isSuccess": true + } + ]} } """) @@ -428,13 +436,52 @@ class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll { actual shouldBe expected } + + it("should fetch vertices using VertexIndex: 'gender in (F, M)') from kakao.user") { + def query(search: String) = { + QueryParser.parse( + s""" + query FetchVertices { + kakao { + user(search: "${search}", ids: ["rain"]) { + id + age + gender + } + } + }""").get + } + + val actualGender = testGraph.queryAsJs(query("gender: M OR gender: F")) + val expected = Json.parse( + """ + { + "data": { + "kakao": { + "user": [ + {"id": "rain", "age": 21, "gender": "T"}, + {"id": "daewon", "age": 20, "gender": "M"}, + {"id": "shon", "age": 19, "gender": "F"} + ] + } + } + } + """) + + actualGender shouldBe expected + + val actualAge = testGraph.queryAsJs(query("age: 19 OR age: 20")) + actualAge shouldBe expected + + val actualAgeBetween = testGraph.queryAsJs(query("age: [10 TO 20]")) + actualAgeBetween shouldBe expected + } } - } - describe("Add edge to label 'friends' and fetch ") { - it("should add edges: daewon -> shon(score: 2) to friends") { - val query = - graphql""" + describe("Add edge to label 'friends' and fetch ") { + it("should add edges: daewon -> shon(score: 2) to friends") { + val query = + graphql""" mutation { addEdge( @@ -449,10 +496,10 @@ class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll { } """ - val actual = testGraph.queryAsJs(query) + val actual = testGraph.queryAsJs(query) - val expected = Json.parse( - """ + val expected = Json.parse( + """ { "data": { "addEdge": [{ @@ -462,12 +509,12 @@ class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll { } """) - actual shouldBe expected - } + actual shouldBe expected + } - it("should fetch edges: friends of kakao.user(daewon) ") { - val query = - graphql""" + it("should fetch edges: friends of kakao.user(daewon) ") { + val query = + graphql""" query { kakao { @@ -495,9 +542,9 @@ class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll { } """ - val actual = testGraph.queryAsJs(query) - val expected = Json.parse( - """ + val actual = testGraph.queryAsJs(query) + val expected = Json.parse( + """ { "data" : { "kakao" : { @@ -526,15 +573,15 @@ class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll { } """) - actual shouldBe expected + actual shouldBe expected + } } - } - describe("Management: Delete label, service column") { + describe("Management: Delete label, service column") { - it("should delete label: 'friends' and serviceColumn: 'user' on kakao") { - val query = - graphql""" + it("should delete label: 'friends' and serviceColumn: 'user' on kakao") { + val query = + graphql""" mutation { Management { @@ -553,9 +600,9 @@ class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll { } """ - val actual = testGraph.queryAsJs(query) - val expected = Json.parse( - """ + val actual = testGraph.queryAsJs(query) + val expected = Json.parse( + """ { "data": { "Management": { @@ -570,12 +617,12 @@ class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll { } """) - actual shouldBe expected - } + actual shouldBe expected + } - it("should fetch failed label: 'friends' and serviceColumn: 'user'") { - val query = - graphql""" + it("should fetch failed label: 'friends' and serviceColumn: 'user'") { + val query = + graphql""" query { Management { @@ -590,13 +637,13 @@ class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll { } """ - // label 'friends' was deleted - assertThrows[ValidationError] { - testGraph.queryAsJs(query) - } + // label 'friends' was deleted + assertThrows[ValidationError] { + testGraph.queryAsJs(query) + } - val queryServiceColumn = - graphql""" + val queryServiceColumn = + graphql""" query { Management { @@ -609,10 +656,10 @@ class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll { } """ - // serviceColumn 'user' was deleted - val actual = testGraph.queryAsJs(queryServiceColumn) - val expected = Json.parse( - """ + // serviceColumn 'user' was deleted + val actual = testGraph.queryAsJs(queryServiceColumn) + val expected = Json.parse( + """ { "data": { "Management": { @@ -624,7 +671,8 @@ class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll { } """) - actual shouldBe expected + actual shouldBe expected + } } } }
