passed S2GraphStructureStandardTest + S2GraphProcessStandardTest.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/4e085e42 Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/4e085e42 Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/4e085e42 Branch: refs/heads/master Commit: 4e085e420a633becfc9ad777bc0c965d7087e2be Parents: b639996 Author: DO YUNG YOON <[email protected]> Authored: Fri Jul 28 17:04:22 2017 +0900 Committer: DO YUNG YOON <[email protected]> Committed: Fri Jul 28 17:04:22 2017 +0900 ---------------------------------------------------------------------- .../core/io/tinkerpop/optimize/S2GraphStep.java | 25 ++++++-- .../scala/org/apache/s2graph/core/S2Graph.scala | 8 +-- .../s2graph/core/index/IndexProvider.scala | 60 +++++++++++++------- .../s2graph/core/index/IndexProviderTest.scala | 31 +++++++++- .../core/tinkerpop/S2GraphProvider.scala | 1 - .../core/tinkerpop/structure/S2GraphTest.scala | 2 +- 6 files changed, 95 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/4e085e42/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java ---------------------------------------------------------------------- diff --git a/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java b/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java index 6773d94..00c277b 100644 --- a/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java +++ b/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java @@ -43,6 +43,7 @@ import org.apache.tinkerpop.gremlin.structure.T; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.util.StringFactory; import java.util.*; +import java.util.stream.Collectors; public class S2GraphStep<S, E extends Element> extends GraphStep<S, E> { private final List<HasContainer> hasContainers = new ArrayList<>(); @@ -88,14 +89,30 @@ public class S2GraphStep<S, E extends Element> extends GraphStep<S, E> { } else { List<VertexId> vids = new ArrayList<>(); List<EdgeId> eids = new ArrayList<>(); + List<HasContainer> filtered = new ArrayList<>(); + + hasContainers.forEach(c -> { + if (c.getKey() == T.id.getAccessor()) { + if (c.getValue() instanceof List<?>) { + ((List<?>)c.getValue()).forEach(o -> { + if (isVertex) vids.add((VertexId) o); + else eids.add((EdgeId) o); + }); + } else { + if (isVertex) vids.add((VertexId)c.getValue()); + else eids.add((EdgeId)c.getValue()); + } + } else { + filtered.add(c); + } + }); if (isVertex) { - List<VertexId> ids = graph.indexProvider().fetchVertexIds(hasContainers); + List<VertexId> ids = graph.indexProvider().fetchVertexIds(filtered.stream().distinct().collect(Collectors.toList())); if (ids.isEmpty()) return (Iterator) graph.vertices(); else return (Iterator) graph.vertices(ids.toArray()); - } - else { - List<EdgeId> ids = graph.indexProvider().fetchEdgeIds(hasContainers); + } else { + List<EdgeId> ids = graph.indexProvider().fetchEdgeIds(filtered.stream().distinct().collect(Collectors.toList())); if (ids.isEmpty()) return (Iterator) graph.edges(); else return (Iterator) graph.edges(ids.toArray()); } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/4e085e42/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala index 3b0a11c..d70651c 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala @@ -67,10 +67,10 @@ object S2Graph { "hbase.table.name" -> "s2graph", "hbase.table.compression.algorithm" -> "gz", "phase" -> "dev", -// "db.default.driver" -> "org.h2.Driver", -// "db.default.url" -> "jdbc:h2:file:./var/metastore;MODE=MYSQL", - "db.default.driver" -> "com.mysql.jdbc.Driver", - "db.default.url" -> "jdbc:mysql://default:3306/graph_dev", + "db.default.driver" -> "org.h2.Driver", + "db.default.url" -> "jdbc:h2:file:./var/metastore;MODE=MYSQL", +// "db.default.driver" -> "com.mysql.jdbc.Driver", +// "db.default.url" -> "jdbc:mysql://default:3306/graph_dev", "db.default.password" -> "graph", "db.default.user" -> "graph", "cache.max.size" -> java.lang.Integer.valueOf(0), http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/4e085e42/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala index e3ab7bc..360ea65 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala @@ -36,6 +36,7 @@ import org.apache.s2graph.core.utils.logger import org.apache.tinkerpop.gremlin.process.traversal.{Compare, Contains, P} import org.apache.tinkerpop.gremlin.process.traversal.step.util.HasContainer import org.apache.tinkerpop.gremlin.process.traversal.util.{AndP, OrP} +import org.apache.tinkerpop.gremlin.structure.T import play.api.libs.json.Json import scala.concurrent.Future @@ -46,6 +47,7 @@ object IndexProvider { val labelField = "_label_" val serviceField = "_service_" val serviceColumnField = "_serviceColumn_" + val hitsPerPage = 100000 def apply(config: Config): IndexProvider = { val indexProviderType = "lucene" @@ -59,23 +61,38 @@ object IndexProvider { def buildQuerySingleString(container: HasContainer): String = { import scala.collection.JavaConversions._ - val key = container.getKey + val key = if (container.getKey == T.label.getAccessor) labelField else container.getKey val value = container.getValue - val biPredicate = container.getBiPredicate - - biPredicate match { - case Contains.within => - key + ":(" + value.asInstanceOf[util.Collection[_]].toSeq.mkString(" OR ") + ")" - case Contains.without => - key + ":NOT (" + value.asInstanceOf[util.Collection[_]].toSeq.mkString(" AND ") + ")" - case Compare.eq => s"${key}:${value}" - case Compare.gte => s"${key}:[${value} TO *] AND NOT ${key}:${value}" - case Compare.gt => s"${key}:[${value} TO *]" - case Compare.lte => s"${key}:[* TO ${value}]" - case Compare.lt => s"${key}:[* TO ${value}] AND NOT ${key}:${value}" - case Compare.neq => s"NOT ${key}:${value}" - case _ => throw new IllegalArgumentException("not supported yet.") + container.getPredicate match { + case and: AndP[_] => + val buffer = scala.collection.mutable.ArrayBuffer.empty[String] + and.getPredicates.foreach { p => + buffer.append(buildQuerySingleString(new HasContainer(container.getKey, p))) + } + buffer.mkString("(", " AND ", ")") + case or: OrP[_] => + val buffer = scala.collection.mutable.ArrayBuffer.empty[String] + or.getPredicates.foreach { p => + buffer.append(buildQuerySingleString(new HasContainer(container.getKey, p))) + } + buffer.mkString("(", " OR ", ")") + case _ => + val biPredicate = container.getBiPredicate + biPredicate match { + + case Contains.within => + key + ":(" + value.asInstanceOf[util.Collection[_]].toSeq.mkString(" OR ") + ")" + case Contains.without => + "NOT " + key + ":(" + value.asInstanceOf[util.Collection[_]].toSeq.mkString(" AND ") + ")" + case Compare.eq => s"${key}:${value}" + case Compare.gt => s"(${key}:[${value} TO *] AND NOT ${key}:${value})" + case Compare.gte => s"${key}:[${value} TO *]" + case Compare.lt => s"${key}:[* TO ${value}]" + case Compare.lte => s"(${key}:[* TO ${value}] OR ${key}:${value})" + case Compare.neq => s"NOT ${key}:${value}" + case _ => throw new IllegalArgumentException("not supported yet.") + } } } @@ -228,14 +245,14 @@ class LuceneIndexProvider(config: Config) extends IndexProvider { override def fetchEdgeIds(hasContainers: java.util.List[HasContainer]): java.util.List[EdgeId] = { val field = eidField - val ids = new java.util.ArrayList[EdgeId] + val ids = new java.util.HashSet[EdgeId] GlobalIndex.findGlobalIndex(hasContainers).map { globalIndex => val queryString = buildQueryString(hasContainers) try { val q = new QueryParser(field, analyzer).parse(queryString) - val hitsPerPage = 10 + val reader = DirectoryReader.open(directories(globalIndex.indexName)) val searcher = new IndexSearcher(reader) @@ -256,18 +273,19 @@ class LuceneIndexProvider(config: Config) extends IndexProvider { } } - ids + new util.ArrayList[EdgeId](ids) } override def fetchVertexIds(hasContainers: java.util.List[HasContainer]): java.util.List[VertexId] = { val field = vidField - val ids = new java.util.ArrayList[VertexId] + val ids = new java.util.HashSet[VertexId] + GlobalIndex.findGlobalIndex(hasContainers).map { globalIndex => val queryString = buildQueryString(hasContainers) try { val q = new QueryParser(field, analyzer).parse(queryString) - val hitsPerPage = 10 + val reader = DirectoryReader.open(directories(globalIndex.indexName)) val searcher = new IndexSearcher(reader) @@ -288,7 +306,7 @@ class LuceneIndexProvider(config: Config) extends IndexProvider { } } - ids + new util.ArrayList[VertexId](ids) } override def shutdown(): Unit = { http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/4e085e42/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala index 7c5d55d..d098458 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala @@ -187,6 +187,35 @@ class IndexProviderTest extends IntegrateCommon { println(s"[[QueryString]: ${queryString}") } - + test("complex.") { + //val predicate = P.lte(10).and(P.between(11, 20)) // .and(P.lt(29).or(P.eq(35))) + val predicate = P.eq(30).and(P.between(11, 20)) // .and(P.lt(29).or(P.eq(35))) + + /* + + (x = 30 and x >= 11 and x < 20) + + // + ((x <= 10 and (x >= 11 and x < 20)) and (x < 29 or x = 35)) + + (and + (lte 10) + (and + (between 11, 20)) + (or + (lt 29) + (eq 35)) + ) + ) + ) + + (x:[* TO 10] OR x:10) AND ((x:[11 TO *] AND x:[* TO 20]) AND (x:[* TO 29] OR x:35)) + ((x:[* TO 10] OR x:10) AND x:[11 TO *] AND x:[* TO 20] AND (x:[* TO 29] OR x:35)) + */ + + val hasContainers = Seq(new HasContainer("x", predicate)) + val queryString = IndexProvider.buildQueryString(hasContainers) + println(s"[[QueryString]: ${queryString}") + } } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/4e085e42/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala index e71ccb3..00768b3 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala @@ -108,7 +108,6 @@ object S2GraphProvider { class S2GraphProvider extends AbstractGraphProvider { override def getBaseConfiguration(s: String, aClass: Class[_], s1: String, graphData: GraphData): util.Map[String, AnyRef] = { - val config = ConfigFactory.load() val m = new java.util.HashMap[String, AnyRef]() m.put(Graph.GRAPH, classOf[S2Graph].getName) // m.put("db.default.url", "jdbc:h2:mem:db1;MODE=MYSQL") http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/4e085e42/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala index 3e618ea..60b48d2 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala @@ -467,7 +467,7 @@ class S2GraphTest extends FunSuite with Matchers with TestCommonWithModels { val e12 = v6.addEdge("created", v3, "weight", Double.box(0.2)) val ls = graph.traversal().E().has("weight", P.eq(Double.box(0.5))) - +// return graph.traversal.V().hasLabel("person").has("age", P.not(P.lte(10).and(P.not(P.between(11, 20)))).and(P.lt(29).or(P.eq(35)))).values("name") val l = ls.toList println(s"[Size]: ${l.size}") println(l.toArray.toSeq.mkString("\n"))
