test more on IndexProviderTest.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/a76dfdc2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/a76dfdc2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/a76dfdc2 Branch: refs/heads/master Commit: a76dfdc207fb26b401f601d45f7aa9e793405a13 Parents: 1cd00df Author: DO YUNG YOON <steams...@apache.org> Authored: Thu Jul 13 10:13:15 2017 +0900 Committer: DO YUNG YOON <steams...@apache.org> Committed: Thu Jul 13 14:20:32 2017 +0900 ---------------------------------------------------------------------- .../core/io/tinkerpop/optimize/S2GraphStep.java | 24 +++--- .../tinkerpop/optimize/S2GraphStepStrategy.java | 1 - .../optimize/S2GraphTraversalUtil.java | 82 ++++++++++---------- .../scala/org/apache/s2graph/core/S2Graph.scala | 16 +++- .../s2graph/core/index/IndexProvider.scala | 6 +- .../s2graph/core/index/IndexProviderTest.scala | 66 +++++++++++----- .../core/tinkerpop/structure/S2GraphTest.scala | 4 +- 7 files changed, 119 insertions(+), 80 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a76dfdc2/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java ---------------------------------------------------------------------- diff --git a/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java b/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java index e769718..f2bc6c8 100644 --- a/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java +++ b/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java @@ -6,6 +6,7 @@ import org.apache.s2graph.core.S2Graph; import org.apache.s2graph.core.index.IndexProvider; import org.apache.s2graph.core.index.IndexProvider$; import org.apache.s2graph.core.mysqls.Label; +import org.apache.s2graph.core.types.VertexId; import org.apache.s2graph.core.utils.logger; import org.apache.tinkerpop.gremlin.process.traversal.Order; import org.apache.tinkerpop.gremlin.process.traversal.Step; @@ -16,16 +17,10 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep; import org.apache.tinkerpop.gremlin.process.traversal.step.map.NoOpBarrierStep; import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.IdentityStep; import org.apache.tinkerpop.gremlin.process.traversal.step.util.HasContainer; -import org.apache.tinkerpop.gremlin.process.traversal.util.MutableMetrics; -import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper; import org.apache.tinkerpop.gremlin.structure.Element; -import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.util.StringFactory; - import java.util.*; -import java.util.function.BiConsumer; -import java.util.function.Consumer; public class S2GraphStep<S, E extends Element> extends GraphStep<S, E> { private final List<HasContainer> hasContainers = new ArrayList<>(); @@ -67,12 +62,19 @@ public class S2GraphStep<S, E extends Element> extends GraphStep<S, E> { String queryString = IndexProvider$.MODULE$.buildQueryString(hasContainers); Boolean isVertex = Vertex.class.isAssignableFrom(this.returnClass); - List<String> ids = new ArrayList<>(); - - if (isVertex) graph.indexProvider().fetchVertexIds(queryString); - else graph.indexProvider().fetchEdgeIds(queryString); + List<VertexId> vids = new ArrayList<>(); + List<EdgeId> eids = new ArrayList<>(); - return (Iterator) (Vertex.class.isAssignableFrom(this.returnClass) ? graph.vertices(ids) : graph.edges(ids)); + if (isVertex) { + List<VertexId> ids = graph.indexProvider().fetchVertexIds(queryString); + if (ids.isEmpty()) return (Iterator) graph.vertices(); + else return (Iterator) graph.vertices(ids); + } + else { + List<EdgeId> ids = graph.indexProvider().fetchEdgeIds(queryString); + if (ids.isEmpty()) return (Iterator) graph.edges(); + else return (Iterator) graph.edges(ids); + } }); } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a76dfdc2/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStepStrategy.java ---------------------------------------------------------------------- diff --git a/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStepStrategy.java b/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStepStrategy.java index e30a502..8f40d15 100644 --- a/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStepStrategy.java +++ b/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStepStrategy.java @@ -30,7 +30,6 @@ public class S2GraphStepStrategy extends AbstractTraversalStrategy<TraversalStra final S2GraphStep<?, ?> s2GraphStep = new S2GraphStep<>(originalGraphStep); TraversalHelper.replaceStep(originalGraphStep, (Step) s2GraphStep, traversal); - } else { Object[] ids = originalGraphStep.getIds(); if (ids[0] instanceof Element) { http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a76dfdc2/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphTraversalUtil.java ---------------------------------------------------------------------- diff --git a/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphTraversalUtil.java b/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphTraversalUtil.java index 72f6999..81a1261 100644 --- a/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphTraversalUtil.java +++ b/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphTraversalUtil.java @@ -1,41 +1,41 @@ -package org.apache.s2graph.core.io.tinkerpop.optimize; - -import org.apache.s2graph.core.S2Vertex; -import org.apache.tinkerpop.gremlin.process.traversal.Step; -import org.apache.tinkerpop.gremlin.process.traversal.Traversal; -import org.apache.tinkerpop.gremlin.process.traversal.Traverser; -import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.IdentityStep; -import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper; -import org.apache.tinkerpop.gremlin.structure.Edge; -import org.apache.tinkerpop.gremlin.structure.Element; -import org.apache.tinkerpop.gremlin.structure.Graph; -import org.apache.tinkerpop.gremlin.structure.Vertex; -import org.apache.tinkerpop.gremlin.structure.util.wrapped.WrappedVertex; - -import java.util.Optional; - -public class S2GraphTraversalUtil { - - public static S2Vertex getS2Vertex(Element v) { - while (v instanceof WrappedVertex) { - v = ((WrappedVertex<Vertex>) v).getBaseVertex(); - } - if (v instanceof S2Vertex) { - return (S2Vertex) v; - } else throw new IllegalArgumentException("Expected traverser of JanusGraph vertex but found: " + v); - } - - public static S2Vertex getS2Vertex(Traverser<? extends Element> traverser) { - return getS2Vertex(traverser.get()); - } - - - public static Step getNextNonIdentityStep(final Step start) { - Step currentStep = start.getNextStep(); - //Skip over identity steps - while (currentStep instanceof IdentityStep) currentStep = currentStep.getNextStep(); - return currentStep; - } - - -} +//package org.apache.s2graph.core.io.tinkerpop.optimize; +// +//import org.apache.s2graph.core.S2Vertex; +//import org.apache.tinkerpop.gremlin.process.traversal.Step; +//import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +//import org.apache.tinkerpop.gremlin.process.traversal.Traverser; +//import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.IdentityStep; +//import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper; +//import org.apache.tinkerpop.gremlin.structure.Edge; +//import org.apache.tinkerpop.gremlin.structure.Element; +//import org.apache.tinkerpop.gremlin.structure.Graph; +//import org.apache.tinkerpop.gremlin.structure.Vertex; +//import org.apache.tinkerpop.gremlin.structure.util.wrapped.WrappedVertex; +// +//import java.util.Optional; +// +//public class S2GraphTraversalUtil { +// +// public static S2Vertex getS2Vertex(Element v) { +// while (v instanceof WrappedVertex) { +// v = ((WrappedVertex<Vertex>) v).getBaseVertex(); +// } +// if (v instanceof S2Vertex) { +// return (S2Vertex) v; +// } else throw new IllegalArgumentException("Expected traverser of JanusGraph vertex but found: " + v); +// } +// +// public static S2Vertex getS2Vertex(Traverser<? extends Element> traverser) { +// return getS2Vertex(traverser.get()); +// } +// +// +// public static Step getNextNonIdentityStep(final Step start) { +// Step currentStep = start.getNextStep(); +// //Skip over identity steps +// while (currentStep instanceof IdentityStep) currentStep = currentStep.getNextStep(); +// return currentStep; +// } +// +// +//} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a76dfdc2/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala index 8a53335..8ee675d 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala @@ -1826,6 +1826,9 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph } addVertexInner(vertex) + indexProvider.mutateVertices(Seq(vertex)) + + vertex } def addVertex(id: VertexId, @@ -1834,11 +1837,14 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph op: Byte = 0, belongLabelIds: Seq[Int] = Seq.empty): S2Vertex = { val vertex = newVertex(id, ts, props, op, belongLabelIds) + val future = mutateVertices(Seq(vertex), withWait = true).map { rets => if (rets.forall(identity)) vertex else throw new RuntimeException("addVertex failed.") } - Await.result(future, WaitTimeout) + Await.ready(future, WaitTimeout) + + vertex } def addVertexInner(vertex: S2Vertex): S2Vertex = { @@ -1846,7 +1852,9 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph if (rets.forall(identity)) vertex else throw new RuntimeException("addVertex failed.") } - Await.result(future, WaitTimeout) + Await.ready(future, WaitTimeout) + + vertex } def addEdge(srcVertex: S2Vertex, labelName: String, tgtVertex: Vertex, kvs: AnyRef*): Edge = { @@ -1876,10 +1884,10 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph val edge = newEdge(srcVertex, otherV, label, dir, op = op, version = ts, propsWithTs = propsWithTs) - indexProvider.mutateEdges(Seq(edge)) - val future = mutateEdges(Seq(edge), withWait = true) Await.ready(future, WaitTimeout) + + indexProvider.mutateEdges(Seq(edge)) edge } catch { case e: LabelNotExistException => throw new java.lang.IllegalArgumentException(e) http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a76dfdc2/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala index 3d5b164..b0d2a4b 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala @@ -99,7 +99,8 @@ class LuceneIndexProvider(config: Config) extends IndexProvider { docs.scoreDocs.foreach { scoreDoc => val document = searcher.doc(scoreDoc.doc) - ids.add(Conversions.s2EdgeIdReads.reads(Json.parse(document.get(field))).get) + val id = Conversions.s2EdgeIdReads.reads(Json.parse(document.get(field))).get +// ids.add(id); } reader.close() @@ -118,7 +119,8 @@ class LuceneIndexProvider(config: Config) extends IndexProvider { docs.scoreDocs.foreach { scoreDoc => val document = searcher.doc(scoreDoc.doc) - ids.add(Conversions.s2VertexIdReads.reads(Json.parse(document.get(field))).get) + val id = Conversions.s2VertexIdReads.reads(Json.parse(document.get(field))).get +// ids.add(id) } reader.close() http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a76dfdc2/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala index f56ac71..b4e5ed2 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala @@ -4,10 +4,11 @@ import org.apache.s2graph.core.Integrate.IntegrateCommon import org.apache.s2graph.core.{Management, S2Vertex} import org.apache.s2graph.core.mysqls._ import org.apache.s2graph.core.types.{InnerVal, InnerValLikeWithTs} - +import scala.collection.JavaConversions._ class IndexProviderTest extends IntegrateCommon { val indexProvider = IndexProvider(config) + val numOfTry = 1 test("test vertex write/query") { import TestUtil._ @@ -28,22 +29,23 @@ class IndexProviderTest extends IntegrateCommon { val vertex = graph.newVertex(vertexId) S2Vertex.fillPropsWithTs(vertex, propsWithTs) - val vertices = Seq(vertex) ++ (0 until 10).map{ ith => - val v = graph.newVertex(vertexId) - S2Vertex.fillPropsWithTs(v, otherPropsWithTs) - v - } + val otherVertex = graph.newVertex(vertexId) + S2Vertex.fillPropsWithTs(otherVertex, otherPropsWithTs) + + val numOfOthers = 10 + val vertices = Seq(vertex) ++ (0 until numOfOthers).map(_ => otherVertex) println(s"[# of vertices]: ${vertices.size}") vertices.foreach(v => println(s"[Vertex]: $v")) indexProvider.mutateVertices(vertices) - import scala.collection.JavaConversions._ - val ids = indexProvider.fetchVertexIds("_timestamp: 1") - ids.head shouldBe vertex.id + (0 until numOfTry).foreach { ith => + var ids = indexProvider.fetchVertexIds("_timestamp: 1") + ids.head shouldBe vertex.id - ids.foreach { id => - println(s"[Id]: $id") + ids.foreach { id => + println(s"[Id]: $id") + } } } test("test edge write/query ") { @@ -64,21 +66,47 @@ class IndexProviderTest extends IntegrateCommon { testLabel.metaPropsInvMap("time") -> InnerValLikeWithTs.withLong(20L, 1L, "v4") ) val edge = graph.newEdge(vertex, vertex, testLabel, 0, propsWithTs = propsWithTs) - val edges = Seq(edge) ++ (0 until 10).map{ ith => - graph.newEdge(otherVertex, otherVertex, testLabel, 0, propsWithTs = otherPropsWithTs) - } + val otherEdge = graph.newEdge(otherVertex, otherVertex, testLabel, 0, propsWithTs = otherPropsWithTs) + val numOfOthers = 10 + val edges = Seq(edge) ++ (0 until numOfOthers).map(_ => otherEdge) println(s"[# of edges]: ${edges.size}") edges.foreach(e => println(s"[Edge]: $e")) indexProvider.mutateEdges(edges) - import scala.collection.JavaConversions._ - val edgeIds = indexProvider.fetchEdgeIds("time: 10 AND _timestamp: 1") - edgeIds.head shouldBe edge.edgeId + // match + (0 until numOfTry).foreach { _ => + + val ids = indexProvider.fetchEdgeIds("time: 10 AND _timestamp: 1") + ids.head shouldBe edge.edgeId - edgeIds.foreach { edgeId => - println(s"[EdgeId]: $edgeId") + ids.foreach { id => + println(s"[Id]: $id") + } } + // match and not + (0 until numOfTry).foreach { _ => + val ids = indexProvider.fetchEdgeIds("time: 20 AND NOT _timestamp: 1") + // ids.size shouldBe 0 + ids.size shouldBe numOfOthers + + ids.foreach { id => + id shouldBe otherEdge.edgeId + println(s"[Id]: $id") + } + } + + // range + (0 until numOfTry).foreach { _ => + val ids = indexProvider.fetchEdgeIds("time: [0 TO 10]") + // ids.size shouldBe 0 + ids.size shouldBe 1 + + ids.foreach { id => + id shouldBe edge.edgeId + println(s"[Id]: $id") + } + } } } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a76dfdc2/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala index 1ef7890..63badc0 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala @@ -466,8 +466,8 @@ class S2GraphTest extends FunSuite with Matchers with TestCommonWithModels { val ls = graph.traversal().V().has("name", "josh") val l = ls.toList - logger.error(s"[Size]: ${l.size}") - logger.error(l.toArray.toSeq.mkString("\n")) + println(s"[Size]: ${l.size}") + println(l.toArray.toSeq.mkString("\n")) println(ls.toList) ls }