add IndexProviderTest.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/1cd00df0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/1cd00df0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/1cd00df0 Branch: refs/heads/master Commit: 1cd00df0a7a58f454182d997c0a8c8337a50486a Parents: 0594fd0 Author: DO YUNG YOON <[email protected]> Authored: Thu Jul 13 07:09:40 2017 +0900 Committer: DO YUNG YOON <[email protected]> Committed: Thu Jul 13 07:09:40 2017 +0900 ---------------------------------------------------------------------- .../core/io/tinkerpop/optimize/S2GraphStep.java | 7 ++- .../s2graph/core/index/IndexProvider.scala | 34 ++++++++--- .../core/Integrate/IntegrateCommon.scala | 2 +- .../s2graph/core/index/IndexProviderTest.scala | 63 +++++++++++++++++--- 4 files changed, 90 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/1cd00df0/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java ---------------------------------------------------------------------- diff --git a/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java b/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java index d494995..e769718 100644 --- a/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java +++ b/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java @@ -66,7 +66,12 @@ public class S2GraphStep<S, E extends Element> extends GraphStep<S, E> { String queryString = IndexProvider$.MODULE$.buildQueryString(hasContainers); Boolean isVertex = Vertex.class.isAssignableFrom(this.returnClass); - List<String> ids = graph.indexProvider().fetchIds(queryString, isVertex); + + List<String> ids = new ArrayList<>(); + + if (isVertex) graph.indexProvider().fetchVertexIds(queryString); + else graph.indexProvider().fetchEdgeIds(queryString); + return (Iterator) (Vertex.class.isAssignableFrom(this.returnClass) ? graph.vertices(ids) : graph.edges(ids)); }); } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/1cd00df0/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala index c8b2ebb..3d5b164 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala @@ -10,7 +10,7 @@ import org.apache.lucene.store.RAMDirectory import org.apache.s2graph.core.io.Conversions import org.apache.s2graph.core.{EdgeId, S2Edge, S2Vertex} import org.apache.s2graph.core.mysqls._ -import org.apache.s2graph.core.types.InnerValLike +import org.apache.s2graph.core.types.{InnerValLike, VertexId} import org.apache.tinkerpop.gremlin.process.traversal.step.util.HasContainer import play.api.libs.json.Json @@ -38,12 +38,14 @@ object IndexProvider { trait IndexProvider { //TODO: Seq nee do be changed into stream - def fetchIds(queryString: String, isVertex: Boolean = true): java.util.List[String] + def fetchEdgeIds(queryString: String): java.util.List[EdgeId] + + def fetchVertexIds(queryString: String): java.util.List[VertexId] def mutateVertices(vertices: Seq[S2Vertex]): Seq[Boolean] def mutateEdges(edges: Seq[S2Edge]): Seq[Boolean] -` + def shutdown(): Unit } @@ -85,9 +87,9 @@ class LuceneIndexProvider(config: Config) extends IndexProvider { edges.map(_ => true) } - override def fetchIds(queryString: String, isVertex: Boolean = true): java.util.List[String] = { - val field = if (isVertex) vidField else eidField - val ids = new java.util.ArrayList[String] + override def fetchEdgeIds(queryString: String): java.util.List[EdgeId] = { + val field = eidField + val ids = new java.util.ArrayList[EdgeId] val q = new QueryParser(field, analyzer).parse(queryString) val hitsPerPage = 10 val reader = DirectoryReader.open(directory) @@ -97,13 +99,31 @@ class LuceneIndexProvider(config: Config) extends IndexProvider { docs.scoreDocs.foreach { scoreDoc => val document = searcher.doc(scoreDoc.doc) - ids.add(document.get(field)) + ids.add(Conversions.s2EdgeIdReads.reads(Json.parse(document.get(field))).get) } reader.close() ids } + override def fetchVertexIds(queryString: String): java.util.List[VertexId] = { + val field = vidField + val ids = new java.util.ArrayList[VertexId] + val q = new QueryParser(field, analyzer).parse(queryString) + val hitsPerPage = 10 + val reader = DirectoryReader.open(directory) + val searcher = new IndexSearcher(reader) + + val docs = searcher.search(q, hitsPerPage) + + docs.scoreDocs.foreach { scoreDoc => + val document = searcher.doc(scoreDoc.doc) + ids.add(Conversions.s2VertexIdReads.reads(Json.parse(document.get(field))).get) + } + + reader.close() + ids + } override def shutdown(): Unit = { writer.close() } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/1cd00df0/s2core/src/test/scala/org/apache/s2graph/core/Integrate/IntegrateCommon.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/IntegrateCommon.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/IntegrateCommon.scala index 3d72432..fd496ee 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/IntegrateCommon.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/IntegrateCommon.scala @@ -20,7 +20,7 @@ package org.apache.s2graph.core.Integrate import com.typesafe.config._ -import org.apache.s2graph.core.mysqls.Label +import org.apache.s2graph.core.mysqls.{Label, Model} import org.apache.s2graph.core.rest.{RequestParser, RestHandler} import org.apache.s2graph.core.utils.logger import org.apache.s2graph.core._ http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/1cd00df0/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala index e9150e9..f56ac71 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala @@ -1,35 +1,84 @@ package org.apache.s2graph.core.index import org.apache.s2graph.core.Integrate.IntegrateCommon +import org.apache.s2graph.core.{Management, S2Vertex} import org.apache.s2graph.core.mysqls._ import org.apache.s2graph.core.types.{InnerVal, InnerValLikeWithTs} -import org.apache.s2graph.core.utils.logger + class IndexProviderTest extends IntegrateCommon { val indexProvider = IndexProvider(config) - test("test write/query ") { + test("test vertex write/query") { + import TestUtil._ +// Management.addVertexProp(testServiceName, testColumnName, "time", "long") + + val testService = Service.findByName(TestUtil.testServiceName).get + val testColumn = ServiceColumn.find(testService.id.get, TestUtil.testColumnName).get + val vertexId = graph.newVertexId(testServiceName)(testColumnName)(1L) + + val propsWithTs = Map( +// testColumn.metasInvMap("time") -> InnerVal.withLong(1L, "v4") + ColumnMeta.timestamp -> InnerVal.withLong(1L, "v4") + ) + val otherPropsWithTs = Map( +// testColumn.metasInvMap("time") -> InnerVal.withLong(2L, "v4") + ColumnMeta.timestamp -> InnerVal.withLong(2L, "v4") + ) + val vertex = graph.newVertex(vertexId) + S2Vertex.fillPropsWithTs(vertex, propsWithTs) + + val vertices = Seq(vertex) ++ (0 until 10).map{ ith => + val v = graph.newVertex(vertexId) + S2Vertex.fillPropsWithTs(v, otherPropsWithTs) + v + } + + println(s"[# of vertices]: ${vertices.size}") + vertices.foreach(v => println(s"[Vertex]: $v")) + indexProvider.mutateVertices(vertices) + + import scala.collection.JavaConversions._ + val ids = indexProvider.fetchVertexIds("_timestamp: 1") + ids.head shouldBe vertex.id + + ids.foreach { id => + println(s"[Id]: $id") + } + } + test("test edge write/query ") { import TestUtil._ val testLabelName = TestUtil.testLabelName val testLabel = Label.findByName(testLabelName).getOrElse(throw new IllegalArgumentException) val vertexId = graph.newVertexId(testServiceName)(testColumnName)(1L) + val otherVertexId = graph.newVertexId(testServiceName)(testColumnName)(2L) val vertex = graph.newVertex(vertexId) + val otherVertex = graph.newVertex(otherVertexId) + val propsWithTs = Map( LabelMeta.timestamp -> InnerValLikeWithTs.withLong(1L, 1L, "v4"), testLabel.metaPropsInvMap("time") -> InnerValLikeWithTs.withLong(10L, 1L, "v4") ) + val otherPropsWithTs = Map( + LabelMeta.timestamp -> InnerValLikeWithTs.withLong(2L, 2L, "v4"), + testLabel.metaPropsInvMap("time") -> InnerValLikeWithTs.withLong(20L, 1L, "v4") + ) val edge = graph.newEdge(vertex, vertex, testLabel, 0, propsWithTs = propsWithTs) - val edges = Seq(edge) + val edges = Seq(edge) ++ (0 until 10).map{ ith => + graph.newEdge(otherVertex, otherVertex, testLabel, 0, propsWithTs = otherPropsWithTs) + } - logger.error(s"[# of edges]: ${edges.size}") - edges.foreach(e => logger.debug(s"[Edge]: $e")) + println(s"[# of edges]: ${edges.size}") + edges.foreach(e => println(s"[Edge]: $e")) indexProvider.mutateEdges(edges) import scala.collection.JavaConversions._ - val edgeIds = indexProvider.fetchIds("time: 10", isVertex = false) + val edgeIds = indexProvider.fetchEdgeIds("time: 10 AND _timestamp: 1") + edgeIds.head shouldBe edge.edgeId edgeIds.foreach { edgeId => - logger.debug(s"[EdgeId]: $edgeId") + println(s"[EdgeId]: $edgeId") } + } }
