add basic test for IndexProvider.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/b639996b Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/b639996b Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/b639996b Branch: refs/heads/master Commit: b639996bf196034fd6bcecc622fd5b12ed61b60a Parents: 33ed1d8 Author: DO YUNG YOON <[email protected]> Authored: Thu Jul 27 20:06:18 2017 +0900 Committer: DO YUNG YOON <[email protected]> Committed: Thu Jul 27 20:06:18 2017 +0900 ---------------------------------------------------------------------- dev_support/graph_mysql/schema.sql | 14 + .../core/io/tinkerpop/optimize/S2GraphStep.java | 6 +- .../org/apache/s2graph/core/mysqls/schema.sql | 35 +-- .../org/apache/s2graph/core/Management.scala | 9 + .../scala/org/apache/s2graph/core/S2Graph.scala | 20 +- .../org/apache/s2graph/core/S2Vertex.scala | 42 --- .../s2graph/core/index/IndexProvider.scala | 266 ++++++++++++++----- .../s2graph/core/mysqls/GlobalIndex.scala | 62 +++++ .../apache/s2graph/core/mysqls/LabelIndex.scala | 55 +++- .../core/Integrate/LabelIndexOptionTest.scala | 144 ---------- .../LabelLabelIndexMutateOptionTest.scala | 144 ++++++++++ .../s2graph/core/index/IndexProviderTest.scala | 73 ++++- .../s2graph/core/models/GlobalIndexTest.scala | 59 ++++ .../core/tinkerpop/S2GraphProvider.scala | 224 ++++++++++------ .../core/tinkerpop/structure/S2GraphTest.scala | 67 ++--- 15 files changed, 799 insertions(+), 421 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b639996b/dev_support/graph_mysql/schema.sql ---------------------------------------------------------------------- diff --git a/dev_support/graph_mysql/schema.sql b/dev_support/graph_mysql/schema.sql index 047f8b2..58c4328 100644 --- a/dev_support/graph_mysql/schema.sql +++ b/dev_support/graph_mysql/schema.sql @@ -121,6 +121,20 @@ CREATE TABLE `labels` ( ALTER TABLE labels add FOREIGN KEY(service_id) REFERENCES services(id); +-- ---------------------------- +-- Table structure for `global_index` +-- ---------------------------- +DROP TABLE IF EXISTS `global_indices`; +CREATE TABLE `global_indices` ( + `id` integer NOT NULL AUTO_INCREMENT, + `prop_names` varchar(255) NOT NULL, + `index_name` varchar(64) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `ux_global_index_index_name` (`index_name`), + UNIQUE KEY `ux_global_index_prop_names` (`prop_names`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + -- ---------------------------- -- Table structure for `label_metas` http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b639996b/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java ---------------------------------------------------------------------- diff --git a/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java b/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java index 783f9b5..6773d94 100644 --- a/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java +++ b/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java @@ -86,18 +86,16 @@ public class S2GraphStep<S, E extends Element> extends GraphStep<S, E> { if (hasContainers.isEmpty()) { return (Iterator) (isVertex ? graph.vertices() : graph.edges()); } else { - String queryString = IndexProvider$.MODULE$.buildQueryString(hasContainers); - List<VertexId> vids = new ArrayList<>(); List<EdgeId> eids = new ArrayList<>(); if (isVertex) { - List<VertexId> ids = graph.indexProvider().fetchVertexIds(queryString); + List<VertexId> ids = graph.indexProvider().fetchVertexIds(hasContainers); if (ids.isEmpty()) return (Iterator) graph.vertices(); else return (Iterator) graph.vertices(ids.toArray()); } else { - List<EdgeId> ids = graph.indexProvider().fetchEdgeIds(queryString); + List<EdgeId> ids = graph.indexProvider().fetchEdgeIds(hasContainers); if (ids.isEmpty()) return (Iterator) graph.edges(); else return (Iterator) graph.edges(ids.toArray()); } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b639996b/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql ---------------------------------------------------------------------- diff --git a/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql b/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql index 51efd0b..521c9d2 100644 --- a/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql +++ b/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql @@ -109,6 +109,18 @@ CREATE TABLE `labels` ( ALTER TABLE labels add FOREIGN KEY(service_id) REFERENCES services(id); +-- ---------------------------- +-- Table structure for `global_index` +-- ---------------------------- +DROP TABLE IF EXISTS `global_indices`; +CREATE TABLE `global_indices` ( + `id` integer NOT NULL AUTO_INCREMENT, + `prop_names` varchar(255) NOT NULL, + `index_name` varchar(64) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `ux_global_index_index_name` (`index_name`), + UNIQUE KEY `ux_global_index_prop_names` (`prop_names`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; -- ---------------------------- @@ -153,29 +165,6 @@ CREATE TABLE `label_indices` ( ALTER TABLE label_indices ADD FOREIGN KEY(label_id) REFERENCES labels(id) ON DELETE CASCADE; - --- ---------------------------- --- Table structure for `service_column_indices` --- ---------------------------- -DROP TABLE IF EXISTS `service_column_indices`; -CREATE TABLE `service_column_indices` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `service_id` int(11) NOT NULL, - `service_column_id` int(11) NOT NULL, - `name` varchar(64) NOT NULL DEFAULT '_PK', - `seq` tinyint(4) NOT NULL, - `meta_seqs` varchar(64) NOT NULL, - `options` text, - PRIMARY KEY (`id`), - UNIQUE KEY `ux_service_id_service_column_id_seq` (`service_id`,`service_column_id`,`seq`), - UNIQUE KEY `ux_service_id_service_column_id_name` (`service_id`, `service_column_id`,`name`), - UNIQUE KEY `ux_service_id_service_column_id_seqs` (`service_id`, `service_column_id`,`meta_seqs`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -ALTER TABLE service_column_indices ADD FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE; -ALTER TABLE service_column_indices ADD FOREIGN KEY(service_column_id) REFERENCES service_columns(id) ON DELETE CASCADE; - - -- ---------------------------- -- Table structure for `experiments` -- ---------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b639996b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/Management.scala b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala index 63a1727..6713baf 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/Management.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala @@ -348,6 +348,15 @@ class Management(graph: S2Graph) { old.consistencyLevel, hTableName, old.hTableTTL, old.schemaVersion, old.isAsync, old.compressionAlgorithm, old.options) } + def buildGlobalIndex(name: String, propNames: Seq[String]): GlobalIndex = { + GlobalIndex.findBy(name) match { + case None => + val idxId = GlobalIndex.insert(name, propNames) + GlobalIndex.findBy(name).get + case Some(oldIndex) => oldIndex + } + } + def getCurrentStorageInfo(labelName: String): Try[Map[String, String]] = for { label <- Try(Label.findByName(labelName, useCache = false).get) } yield { http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b639996b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala index 318c092..3b0a11c 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala @@ -67,8 +67,10 @@ object S2Graph { "hbase.table.name" -> "s2graph", "hbase.table.compression.algorithm" -> "gz", "phase" -> "dev", - "db.default.driver" -> "org.h2.Driver", - "db.default.url" -> "jdbc:h2:file:./var/metastore;MODE=MYSQL", +// "db.default.driver" -> "org.h2.Driver", +// "db.default.url" -> "jdbc:h2:file:./var/metastore;MODE=MYSQL", + "db.default.driver" -> "com.mysql.jdbc.Driver", + "db.default.url" -> "jdbc:mysql://default:3306/graph_dev", "db.default.password" -> "graph", "db.default.user" -> "graph", "cache.max.size" -> java.lang.Integer.valueOf(0), @@ -1826,7 +1828,6 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph } addVertexInner(vertex) - indexProvider.mutateVertices(Seq(vertex)) vertex } @@ -1848,15 +1849,17 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph } def addVertexInner(vertex: S2Vertex): S2Vertex = { - val future = mutateVertices(Seq(vertex), withWait = true).map { rets => - if (rets.forall(identity)) vertex - else throw new RuntimeException("addVertex failed.") + val future = mutateVertices(Seq(vertex), withWait = true).flatMap { rets => + if (rets.forall(identity)) { + indexProvider.mutateVerticesAsync(Seq(vertex)) + } else throw new RuntimeException("addVertex failed.") } Await.ready(future, WaitTimeout) vertex } + /* tp3 only */ def addEdge(srcVertex: S2Vertex, labelName: String, tgtVertex: Vertex, kvs: AnyRef*): Edge = { val containsId = kvs.contains(T.id) @@ -1884,10 +1887,11 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph val edge = newEdge(srcVertex, otherV, label, dir, op = op, version = ts, propsWithTs = propsWithTs) - val future = mutateEdges(Seq(edge), withWait = true) + val future = mutateEdges(Seq(edge), withWait = true).flatMap { rets => + indexProvider.mutateEdgesAsync(Seq(edge)) + } Await.ready(future, WaitTimeout) - indexProvider.mutateEdges(Seq(edge)) edge } catch { case e: LabelNotExistException => throw new java.lang.IllegalArgumentException(e) http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b639996b/s2core/src/main/scala/org/apache/s2graph/core/S2Vertex.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Vertex.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Vertex.scala index d485a01..c0dc23b 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/S2Vertex.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Vertex.scala @@ -201,48 +201,6 @@ case class S2Vertex(graph: S2Graph, override def addEdge(labelName: String, vertex: Vertex, kvs: AnyRef*): Edge = { graph.addEdge(this, labelName, vertex, kvs: _*) -// val containsId = kvs.contains(T.id) -// vertex match { -// case otherV: S2Vertex => -// if (!graph.features().edge().supportsUserSuppliedIds() && containsId) { -// throw Exceptions.userSuppliedIdsNotSupported() -// } -// -// val props = S2Property.kvsToProps(kvs) -// -// props.foreach { case (k, v) => S2Property.assertValidProp(k, v) } -// -// //TODO: direction, operation, _timestamp need to be reserved property key. -// -// try { -// val direction = props.get("direction").getOrElse("out").toString -// val ts = props.get(LabelMeta.timestamp.name).map(_.toString.toLong).getOrElse(System.currentTimeMillis()) -// val operation = props.get("operation").map(_.toString).getOrElse("insert") -// val label = Label.findByName(labelName).getOrElse(throw new LabelNotExistException(labelName)) -// val dir = GraphUtil.toDir(direction).getOrElse(throw new RuntimeException(s"$direction is not supported.")) -// val propsPlusTs = props ++ Map(LabelMeta.timestamp.name -> ts) -// val propsWithTs = label.propsToInnerValsWithTs(propsPlusTs, ts) -// val op = GraphUtil.toOp(operation).getOrElse(throw new RuntimeException(s"$operation is not supported.")) -// -// val edge = graph.newEdge(this, otherV, label, dir, op = op, version = ts, propsWithTs = propsWithTs) -//// //TODO: return type of mutateEdges can contains information if snapshot edge already exist. -//// // instead call checkEdges, we can exploit this feature once we refactor return type. -//// implicit val ec = graph.ec -//// val future = graph.checkEdges(Seq(edge)).flatMap { stepResult => -//// if (stepResult.edgeWithScores.nonEmpty) -//// Future.failed(throw Graph.Exceptions.edgeWithIdAlreadyExists(edge.id())) -//// else -//// graph.mutateEdges(Seq(edge), withWait = true) -//// } -// val future = graph.mutateEdges(Seq(edge), withWait = true) -// Await.ready(future, graph.WaitTimeout) -// edge -// } catch { -// case e: LabelNotExistException => throw new java.lang.IllegalArgumentException(e) -// } -// case null => throw new java.lang.IllegalArgumentException -// case _ => throw new RuntimeException("only S2Graph vertex can be used.") -// } } override def property[V](key: String): VertexProperty[V] = { http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b639996b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala index ed97ed9..e3ab7bc 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -23,17 +23,19 @@ import java.util import com.typesafe.config.Config import org.apache.lucene.analysis.standard.StandardAnalyzer -import org.apache.lucene.document.{Document, Field, StringField, TextField} +import org.apache.lucene.document._ import org.apache.lucene.index.{DirectoryReader, IndexWriter, IndexWriterConfig} import org.apache.lucene.queryparser.classic.{ParseException, QueryParser} import org.apache.lucene.search.IndexSearcher -import org.apache.lucene.store.RAMDirectory +import org.apache.lucene.store.{BaseDirectory, RAMDirectory} import org.apache.s2graph.core.io.Conversions import org.apache.s2graph.core.{EdgeId, S2Edge, S2Vertex} import org.apache.s2graph.core.mysqls._ import org.apache.s2graph.core.types.{InnerValLike, VertexId} import org.apache.s2graph.core.utils.logger +import org.apache.tinkerpop.gremlin.process.traversal.{Compare, Contains, P} import org.apache.tinkerpop.gremlin.process.traversal.step.util.HasContainer +import org.apache.tinkerpop.gremlin.process.traversal.util.{AndP, OrP} import play.api.libs.json.Json import scala.concurrent.Future @@ -41,6 +43,9 @@ import scala.concurrent.Future object IndexProvider { val vidField = "_vid_" val eidField = "_eid_" + val labelField = "_label_" + val serviceField = "_service_" + val serviceColumnField = "_serviceColumn_" def apply(config: Config): IndexProvider = { val indexProviderType = "lucene" @@ -51,22 +56,63 @@ object IndexProvider { } } - def buildQueryString(hasContainers: java.util.List[HasContainer]): String = { + def buildQuerySingleString(container: HasContainer): String = { import scala.collection.JavaConversions._ - hasContainers.map { container => - container.getKey + ":" + container.getValue - }.mkString(" AND ") + + val key = container.getKey + val value = container.getValue + + val biPredicate = container.getBiPredicate + + biPredicate match { + case Contains.within => + key + ":(" + value.asInstanceOf[util.Collection[_]].toSeq.mkString(" OR ") + ")" + case Contains.without => + key + ":NOT (" + value.asInstanceOf[util.Collection[_]].toSeq.mkString(" AND ") + ")" + case Compare.eq => s"${key}:${value}" + case Compare.gte => s"${key}:[${value} TO *] AND NOT ${key}:${value}" + case Compare.gt => s"${key}:[${value} TO *]" + case Compare.lte => s"${key}:[* TO ${value}]" + case Compare.lt => s"${key}:[* TO ${value}] AND NOT ${key}:${value}" + case Compare.neq => s"NOT ${key}:${value}" + case _ => throw new IllegalArgumentException("not supported yet.") + } } + def buildQueryString(hasContainers: java.util.List[HasContainer]): String = { + import scala.collection.JavaConversions._ + val builder = scala.collection.mutable.ArrayBuffer.empty[String] + + hasContainers.foreach { container => + container.getPredicate match { + case and: AndP[_] => + val buffer = scala.collection.mutable.ArrayBuffer.empty[String] + and.getPredicates.foreach { p => + buffer.append(buildQuerySingleString(new HasContainer(container.getKey, p))) + } + builder.append(buffer.mkString("(", " AND ", ")")) + case or: OrP[_] => + val buffer = scala.collection.mutable.ArrayBuffer.empty[String] + or.getPredicates.foreach { p => + buffer.append(buildQuerySingleString(new HasContainer(container.getKey, p))) + } + builder.append(buffer.mkString("(", " OR ", ")")) + case _ => + builder.append(buildQuerySingleString(container)) + } + } + + builder.mkString(" AND ") + } } trait IndexProvider { //TODO: Seq nee do be changed into stream - def fetchEdgeIds(queryString: String): java.util.List[EdgeId] - def fetchEdgeIdsAsync(queryString: String): Future[java.util.List[EdgeId]] + def fetchEdgeIds(hasContainers: java.util.List[HasContainer]): java.util.List[EdgeId] + def fetchEdgeIdsAsync(hasContainers: java.util.List[HasContainer]): Future[java.util.List[EdgeId]] - def fetchVertexIds(queryString: String): java.util.List[VertexId] - def fetchVertexIdsAsync(queryString: String): Future[java.util.List[VertexId]] + def fetchVertexIds(hasContainers: java.util.List[HasContainer]): java.util.List[VertexId] + def fetchVertexIdsAsync(hasContainers: java.util.List[HasContainer]): Future[java.util.List[VertexId]] def mutateVertices(vertices: Seq[S2Vertex]): Seq[Boolean] def mutateVerticesAsync(vertices: Seq[S2Vertex]): Future[Seq[Boolean]] @@ -79,101 +125,179 @@ trait IndexProvider { class LuceneIndexProvider(config: Config) extends IndexProvider { import IndexProvider._ + import scala.collection.mutable + import scala.collection.JavaConverters._ val analyzer = new StandardAnalyzer() - val directory = new RAMDirectory() - val indexConfig = new IndexWriterConfig(analyzer) - val writer = new IndexWriter(directory, indexConfig) + val writers = mutable.Map.empty[String, IndexWriter] + val directories = mutable.Map.empty[String, BaseDirectory] + + private def getOrElseCreateIndexWriter(indexName: String): IndexWriter = { + writers.getOrElseUpdate(indexName, { + val dir = directories.getOrElseUpdate(indexName, new RAMDirectory()) + val indexConfig = new IndexWriterConfig(analyzer) + new IndexWriter(dir, indexConfig) + }) + } - override def mutateVertices(vertices: Seq[S2Vertex]): Seq[Boolean] = { - vertices.map { vertex => + private def toDocument(globalIndex: GlobalIndex, vertex: S2Vertex): Option[Document] = { + val props = vertex.props.asScala + val exist = props.exists(t => globalIndex.propNamesSet(t._1)) + if (!exist) None + else { val doc = new Document() - val id = vertex.id.toString() - doc.add(new StringField(vidField, id, Field.Store.YES)) + val id = vertex.id.toString - vertex.properties.foreach { case (dim, value) => - doc.add(new TextField(dim, value.toString, Field.Store.YES)) + doc.add(new StringField(vidField, id, Field.Store.YES)) + doc.add(new StringField(serviceField, vertex.serviceName, Field.Store.YES)) + doc.add(new StringField(serviceColumnField, vertex.columnName, Field.Store.YES)) + + props.foreach { case (dim, s2VertexProperty) => + val shouldIndex = if (globalIndex.propNamesSet(dim)) Field.Store.YES else Field.Store.NO + val field = s2VertexProperty.columnMeta.dataType match { + case "string" => new StringField(dim, s2VertexProperty.innerVal.value.toString, shouldIndex) + case _ => new StringField(dim, s2VertexProperty.innerVal.value.toString, shouldIndex) + } + doc.add(field) } - writer.addDocument(doc) + + Option(doc) } - writer.commit() - vertices.map(_ => true) } - override def mutateEdges(edges: Seq[S2Edge]): Seq[Boolean] = { - edges.map { edge => + private def toDocument(globalIndex: GlobalIndex, edge: S2Edge): Option[Document] = { + val props = edge.propsWithTs.asScala + val exist = props.exists(t => globalIndex.propNamesSet(t._1)) + if (!exist) None + else { val doc = new Document() val id = edge.edgeId.toString + doc.add(new StringField(eidField, id, Field.Store.YES)) + doc.add(new StringField(serviceField, edge.serviceName, Field.Store.YES)) + doc.add(new StringField(labelField, edge.label(), Field.Store.YES)) + + props.foreach { case (dim, s2Property) => + val shouldIndex = if (globalIndex.propNamesSet(dim)) Field.Store.YES else Field.Store.NO + val field = s2Property.labelMeta.dataType match { + case "string" => new StringField(dim, s2Property.innerVal.value.toString, shouldIndex) + case _ => new StringField(dim, s2Property.innerVal.value.toString, shouldIndex) + } + doc.add(field) + } + + Option(doc) + } + } + + override def mutateVertices(vertices: Seq[S2Vertex]): Seq[Boolean] = { + val globalIndexOptions = GlobalIndex.findAll() + + globalIndexOptions.map { globalIndex => + val writer = getOrElseCreateIndexWriter(globalIndex.indexName) - edge.properties.foreach { case (dim, value) => - doc.add(new TextField(dim, value.toString, Field.Store.YES)) + vertices.foreach { vertex => + toDocument(globalIndex, vertex).foreach { doc => + writer.addDocument(doc) + } } - writer.addDocument(doc) + + writer.commit() } - writer.commit() + + vertices.map(_ => true) + } + + override def mutateEdges(edges: Seq[S2Edge]): Seq[Boolean] = { + val globalIndexOptions = GlobalIndex.findAll() + + globalIndexOptions.map { globalIndex => + val writer = getOrElseCreateIndexWriter(globalIndex.indexName) + + edges.foreach { edge => + toDocument(globalIndex, edge).foreach { doc => + writer.addDocument(doc) + } + } + + writer.commit() + } + edges.map(_ => true) } - override def fetchEdgeIds(queryString: String): java.util.List[EdgeId] = { + override def fetchEdgeIds(hasContainers: java.util.List[HasContainer]): java.util.List[EdgeId] = { val field = eidField val ids = new java.util.ArrayList[EdgeId] - try { - val q = new QueryParser(field, analyzer).parse(queryString) - val hitsPerPage = 10 - val reader = DirectoryReader.open(directory) - val searcher = new IndexSearcher(reader) - - val docs = searcher.search(q, hitsPerPage) - - docs.scoreDocs.foreach { scoreDoc => - val document = searcher.doc(scoreDoc.doc) - val id = Conversions.s2EdgeIdReads.reads(Json.parse(document.get(field))).get - ids.add(id); - } - reader.close() - ids - } catch { - case ex: ParseException => - logger.error(s"[IndexProvider]: ${queryString} parse failed.", ex) + GlobalIndex.findGlobalIndex(hasContainers).map { globalIndex => + val queryString = buildQueryString(hasContainers) + + try { + val q = new QueryParser(field, analyzer).parse(queryString) + val hitsPerPage = 10 + val reader = DirectoryReader.open(directories(globalIndex.indexName)) + val searcher = new IndexSearcher(reader) + + val docs = searcher.search(q, hitsPerPage) + + docs.scoreDocs.foreach { scoreDoc => + val document = searcher.doc(scoreDoc.doc) + val id = Conversions.s2EdgeIdReads.reads(Json.parse(document.get(field))).get + ids.add(id); + } + + reader.close() ids + } catch { + case ex: ParseException => + logger.error(s"[IndexProvider]: ${queryString} parse failed.", ex) + ids + } } + ids } - override def fetchVertexIds(queryString: String): java.util.List[VertexId] = { + override def fetchVertexIds(hasContainers: java.util.List[HasContainer]): java.util.List[VertexId] = { val field = vidField val ids = new java.util.ArrayList[VertexId] - try { - val q = new QueryParser(field, analyzer).parse(queryString) - val hitsPerPage = 10 - val reader = DirectoryReader.open(directory) - val searcher = new IndexSearcher(reader) - - val docs = searcher.search(q, hitsPerPage) - - docs.scoreDocs.foreach { scoreDoc => - val document = searcher.doc(scoreDoc.doc) - val id = Conversions.s2VertexIdReads.reads(Json.parse(document.get(field))).get - ids.add(id) - } + GlobalIndex.findGlobalIndex(hasContainers).map { globalIndex => + val queryString = buildQueryString(hasContainers) - reader.close() - ids - } catch { - case ex: ParseException => - logger.error(s"[IndexProvider]: ${queryString} parse failed.", ex) + try { + val q = new QueryParser(field, analyzer).parse(queryString) + val hitsPerPage = 10 + val reader = DirectoryReader.open(directories(globalIndex.indexName)) + val searcher = new IndexSearcher(reader) + + val docs = searcher.search(q, hitsPerPage) + + docs.scoreDocs.foreach { scoreDoc => + val document = searcher.doc(scoreDoc.doc) + val id = Conversions.s2VertexIdReads.reads(Json.parse(document.get(field))).get + ids.add(id) + } + + reader.close() ids + } catch { + case ex: ParseException => + logger.error(s"[IndexProvider]: ${queryString} parse failed.", ex) + ids + } } + + ids } + override def shutdown(): Unit = { - writer.close() + writers.foreach { case (_, writer) => writer.close() } } - override def fetchEdgeIdsAsync(queryString: String): Future[util.List[EdgeId]] = Future.successful(fetchEdgeIds(queryString)) + override def fetchEdgeIdsAsync(hasContainers: java.util.List[HasContainer]): Future[util.List[EdgeId]] = Future.successful(fetchEdgeIds(hasContainers)) - override def fetchVertexIdsAsync(queryString: String): Future[util.List[VertexId]] = Future.successful(fetchVertexIds(queryString)) + override def fetchVertexIdsAsync(hasContainers: java.util.List[HasContainer]): Future[util.List[VertexId]] = Future.successful(fetchVertexIds(hasContainers)) override def mutateVerticesAsync(vertices: Seq[S2Vertex]): Future[Seq[Boolean]] = Future.successful(mutateVertices(vertices)) http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b639996b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/GlobalIndex.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/GlobalIndex.scala b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/GlobalIndex.scala new file mode 100644 index 0000000..69323b7 --- /dev/null +++ b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/GlobalIndex.scala @@ -0,0 +1,62 @@ +package org.apache.s2graph.core.mysqls + +import org.apache.tinkerpop.gremlin.process.traversal.step.util.HasContainer +import scalikejdbc.{AutoSession, DBSession, WrappedResultSet} +import scalikejdbc._ + +object GlobalIndex extends Model[GlobalIndex] { + import org.apache.s2graph.core.index.IndexProvider._ + + val DefaultIndexName = GlobalIndex(None, Seq(vidField, eidField, serviceField, serviceColumnField, labelField), "_default_") + + val TableName = "global_indices" + + def apply(rs: WrappedResultSet): GlobalIndex = { + GlobalIndex(rs.intOpt("id"), rs.string("prop_names").split(",").sorted, rs.string("index_name")) + } + + def findBy(indexName: String, useCache: Boolean = true)(implicit session: DBSession = AutoSession): Option[GlobalIndex] = { + val cacheKey = s"indexName=$indexName" + lazy val sql = sql"""select * from global_indices where index_name = $indexName""".map { rs => GlobalIndex(rs) }.single.apply() + if (useCache) withCache(cacheKey){sql} + else sql + } + + def insert(indexName: String, propNames: Seq[String])(implicit session: DBSession = AutoSession): Long = { + sql"""insert into global_indices(prop_names, index_name) values(${propNames.sorted.mkString(",")}, $indexName)""" + .updateAndReturnGeneratedKey.apply() + } + + def findAll(useCache: Boolean = true)(implicit session: DBSession = AutoSession): Seq[GlobalIndex] = { + lazy val ls = sql"""select * from global_indices """.map { rs => GlobalIndex(rs) }.list.apply + if (useCache) { + listCache.withCache("findAll") { + putsToCache(ls.map { globalIndex => + val cacheKey = s"indexName=${globalIndex.indexName}" + cacheKey -> globalIndex + }) + ls + } + } else { + ls + } + } + + def findGlobalIndex(hasContainers: java.util.List[HasContainer])(implicit session: DBSession = AutoSession): Option[GlobalIndex] = { + import scala.collection.JavaConversions._ + val indices = findAll(useCache = true) + val keys = hasContainers.map(_.getKey) + + val sorted = indices.map { index => + val matched = keys.filter(index.propNamesSet) + index -> matched.length + }.filter(_._2 > 0).sortBy(_._2 * -1) + + sorted.headOption.map(_._1) + } + +} + +case class GlobalIndex(id: Option[Int], propNames: Seq[String], indexName: String) { + lazy val propNamesSet = propNames.toSet +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b639996b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/LabelIndex.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/LabelIndex.scala b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/LabelIndex.scala index e82e502..1da0e55 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/LabelIndex.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/LabelIndex.scala @@ -20,7 +20,7 @@ package org.apache.s2graph.core.mysqls import org.apache.s2graph.core.GraphUtil -import org.apache.s2graph.core.mysqls.LabelIndex.IndexOption +import org.apache.s2graph.core.mysqls.LabelIndex.LabelIndexMutateOption import org.apache.s2graph.core.utils.logger import play.api.libs.json.{JsObject, JsString, Json} import scalikejdbc._ @@ -42,11 +42,11 @@ object LabelIndex extends Model[LabelIndex] { ) } - case class IndexOption(dir: Byte, - method: String, - rate: Double, - totalModular: Long, - storeDegree: Boolean) { + case class LabelIndexMutateOption(dir: Byte, + method: String, + rate: Double, + totalModular: Long, + storeDegree: Boolean) { val isBufferIncrement = method == "drop" || method == "sample" || method == "hash_sample" @@ -169,7 +169,44 @@ object LabelIndex extends Model[LabelIndex] { }.toList) } } - +/** +mgmt.buildIndex('nameAndAge',Vertex.class) +.addKey(name,Mapping.TEXT.getParameter()) +.addKey(age,Mapping.TEXT.getParameter()) +.buildMixedIndex("search") + +v: {name: abc} - E1: {age: 20}, E2, E3.... + +Management.createServiceColumn( + serviceName = serviceName, columnName = "person", columnType = "integer", + props = Seq( + Prop("name", "-", "string"), + Prop("age", "0", "integer"), + Prop("location", "-", "string") + ) +) + +management.createLabel( + label = "bought", + srcServiceName = serviceName, srcColumnName = "person", srcColumnType = "integer", + tgtServiceName = serviceName, tgtColumnName = "product", tgtColumnType = "integer", idDirected = true, + serviceName = serviceName, + indices = Seq( + Index("PK", Seq("amount", "created_at"), IndexType("mixed", propsMapping: Map[String, String]), +{"in": {}, "out": {}}) + ), + props = Seq( + Prop("amount", "0.0", "double"), + Prop("created_at", "2000-01-01", "string") + ), + consistencyLevel = "strong" +) + +mgmt.buildIndex('PK', Edge.class) + .addKey(amount, Double) + .buildCompositeIndex + +*/ case class LabelIndex(id: Option[Int], labelId: Int, name: String, seq: Byte, metaSeqs: Seq[Byte], formulars: String, dir: Option[Int], options: Option[String]) { // both @@ -191,7 +228,7 @@ case class LabelIndex(id: Option[Int], labelId: Int, name: String, seq: Byte, me ) } - def parseOption(dir: String): Option[IndexOption] = try { + def parseOption(dir: String): Option[LabelIndexMutateOption] = try { options.map { string => val jsObj = Json.parse(string) \ dir @@ -200,7 +237,7 @@ case class LabelIndex(id: Option[Int], labelId: Int, name: String, seq: Byte, me val totalModular = (jsObj \ "totalModular").asOpt[Long].getOrElse(100L) val storeDegree = (jsObj \ "storeDegree").asOpt[Boolean].getOrElse(true) - IndexOption(GraphUtil.directions(dir).toByte, method, rate, totalModular, storeDegree) + LabelIndexMutateOption(GraphUtil.directions(dir).toByte, method, rate, totalModular, storeDegree) } } catch { case e: Exception => http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b639996b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/LabelIndexOptionTest.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/LabelIndexOptionTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/LabelIndexOptionTest.scala deleted file mode 100644 index 6e213b0..0000000 --- a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/LabelIndexOptionTest.scala +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.s2graph.core.Integrate - -import org.apache.s2graph.core._ -import org.scalatest.BeforeAndAfterEach -import play.api.libs.json._ - -class LabelIndexOptionTest extends IntegrateCommon with BeforeAndAfterEach { - - import TestUtil._ - - // called by start test, once - override def initTestData(): Unit = { - super.initTestData() - - val insert = "insert" - val e = "e" - val weight = "weight" - val is_hidden = "is_hidden" - - insertEdgesSync( - toEdge(1, insert, e, 0, 1, testLabelNameLabelIndex), - toEdge(1, insert, e, 0, 2, testLabelNameLabelIndex), - toEdge(1, insert, e, 0, 3, testLabelNameLabelIndex) - ) - } - - def getQuery(ids: Seq[Int], direction: String, indexName: String): Query = - Query( - vertices = ids.map(graph.toVertex(testServiceName, testColumnName, _)), - steps = Vector( - Step(Seq(QueryParam(testLabelNameLabelIndex, direction = direction, indexName = indexName))) - ) - ) - - /** - * "indices": [ - * {"name": "$index1", "propNames": ["weight", "time", "is_hidden", "is_blocked"]}, - * {"name": "$idxStoreInDropDegree", "propNames": ["time"], "options": { "in": {"storeDegree": false }, "out": {"method": "drop", "storeDegree": false }}}, - * {"name": "$idxStoreOutDropDegree", "propNames": ["weight"], "options": { "out": {"storeDegree": false}, "in": { "method": "drop", "storeDegree": false }}}, - * {"name": "$idxStoreIn", "propNames": ["is_hidden"], "options": { "out": {"method": "drop", "storeDegree": false }}}, - * {"name": "$idxStoreOut", "propNames": ["weight", "is_blocked"], "options": { "in": {"method": "drop", "storeDegree": false }, "out": {"method": "normal" }}}, - * {"name": "$idxDropInStoreDegree", "propNames": ["is_blocked"], "options": { "in": {"method": "drop" }, "out": {"method": "drop", "storeDegree": false }}}, - * {"name": "$idxDropOutStoreDegree", "propNames": ["weight", "is_blocked", "_timestamp"], "options": { "in": {"method": "drop", "storeDegree": false }, "out": {"method": "drop"}}} - * ], - **/ - - /** - * index without no options - */ - test("normal index should store in/out direction Edges with Degrees") { - var edges = getEdgesSync(getQuery(Seq(0), "out", index1)) - (edges \ "results").as[Seq[JsValue]].size should be(3) - (edges \\ "_degree").map(_.as[Long]).sum should be(3) - - edges = getEdgesSync(getQuery(Seq(1, 2, 3), "in", index1)) - (edges \ "results").as[Seq[JsValue]].size should be(3) - (edges \\ "_degree").map(_.as[Long]).sum should be(3) - } - - /** - * { "out": {"method": "drop", "storeDegree": false } } - */ - test("storeDegree: store out direction Edge and drop Degree") { - val edges = getEdgesSync(getQuery(Seq(0), "out", idxStoreOutDropDegree)) - (edges \ "results").as[Seq[JsValue]].size should be(3) - (edges \\ "_degree").map(_.as[Long]).sum should be(0) - } - - /** - * { "in": { "method": "drop", "storeDegree": false } } - */ - test("storeDegree: store in direction Edge and drop Degree") { - val edges = getEdgesSync(getQuery(Seq(1, 2, 3), "in", idxStoreInDropDegree)) - (edges \ "results").as[Seq[JsValue]].size should be(3) - (edges \\ "_degree").map(_.as[Long]).sum should be(0) - } - - /** - * { "out": {"method": "drop", "storeDegree": false } } - */ - test("index for in direction should drop out direction edge") { - val edges = getEdgesSync(getQuery(Seq(0), "out", idxStoreIn)) - (edges \ "results").as[Seq[JsValue]].size should be(0) - (edges \\ "_degree").map(_.as[Long]).sum should be(0) - } - - test("index for in direction should store in direction edge") { - val edges = getEdgesSync(getQuery(Seq(1, 2, 3), "in", idxStoreIn)) - (edges \ "results").as[Seq[JsValue]].size should be(3) - (edges \\ "_degree").map(_.as[Long]).sum should be(3) - } - - /** - * { "in": {"method": "drop", "storeDegree": false } } - */ - test("index for out direction should store out direction edge") { - val edges = getEdgesSync(getQuery(Seq(0), "out", idxStoreOut)) - (edges \ "results").as[Seq[JsValue]].size should be(3) - (edges \\ "_degree").map(_.as[Long]).sum should be(3) - } - - test("index for out direction should drop in direction edge") { - val edges = getEdgesSync(getQuery(Seq(1, 2, 3), "in", idxStoreOut)) - (edges \ "results").as[Seq[JsValue]].size should be(0) - (edges \\ "_degree").map(_.as[Long]).sum should be(0) - } - - /** - * { "out": {"method": "drop", "storeDegree": false} } - */ - test("index for in direction should drop in direction edge and store degree") { - val edges = getEdgesSync(getQuery(Seq(1, 2, 3), "in", idxDropInStoreDegree)) - (edges \ "results").as[Seq[JsValue]].size should be(0) - (edges \\ "_degree").map(_.as[Long]).sum should be(3) - } - - /** - * { "in": {"method": "drop", "storeDegree": false }, "out": {"method": "drop"} } - */ - test("index for out direction should drop out direction edge and store degree") { - val edges = getEdgesSync(getQuery(Seq(0), "out", idxDropOutStoreDegree)) - (edges \ "results").as[Seq[JsValue]].size should be(0) - (edges \\ "_degree").map(_.as[Long]).sum should be(3) - } -} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b639996b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/LabelLabelIndexMutateOptionTest.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/LabelLabelIndexMutateOptionTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/LabelLabelIndexMutateOptionTest.scala new file mode 100644 index 0000000..4855cfc --- /dev/null +++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/LabelLabelIndexMutateOptionTest.scala @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.s2graph.core.Integrate + +import org.apache.s2graph.core._ +import org.scalatest.BeforeAndAfterEach +import play.api.libs.json._ + +class LabelLabelIndexMutateOptionTest extends IntegrateCommon with BeforeAndAfterEach { + + import TestUtil._ + + // called by start test, once + override def initTestData(): Unit = { + super.initTestData() + + val insert = "insert" + val e = "e" + val weight = "weight" + val is_hidden = "is_hidden" + + insertEdgesSync( + toEdge(1, insert, e, 0, 1, testLabelNameLabelIndex), + toEdge(1, insert, e, 0, 2, testLabelNameLabelIndex), + toEdge(1, insert, e, 0, 3, testLabelNameLabelIndex) + ) + } + + def getQuery(ids: Seq[Int], direction: String, indexName: String): Query = + Query( + vertices = ids.map(graph.toVertex(testServiceName, testColumnName, _)), + steps = Vector( + Step(Seq(QueryParam(testLabelNameLabelIndex, direction = direction, indexName = indexName))) + ) + ) + + /** + * "indices": [ + * {"name": "$index1", "propNames": ["weight", "time", "is_hidden", "is_blocked"]}, + * {"name": "$idxStoreInDropDegree", "propNames": ["time"], "options": { "in": {"storeDegree": false }, "out": {"method": "drop", "storeDegree": false }}}, + * {"name": "$idxStoreOutDropDegree", "propNames": ["weight"], "options": { "out": {"storeDegree": false}, "in": { "method": "drop", "storeDegree": false }}}, + * {"name": "$idxStoreIn", "propNames": ["is_hidden"], "options": { "out": {"method": "drop", "storeDegree": false }}}, + * {"name": "$idxStoreOut", "propNames": ["weight", "is_blocked"], "options": { "in": {"method": "drop", "storeDegree": false }, "out": {"method": "normal" }}}, + * {"name": "$idxDropInStoreDegree", "propNames": ["is_blocked"], "options": { "in": {"method": "drop" }, "out": {"method": "drop", "storeDegree": false }}}, + * {"name": "$idxDropOutStoreDegree", "propNames": ["weight", "is_blocked", "_timestamp"], "options": { "in": {"method": "drop", "storeDegree": false }, "out": {"method": "drop"}}} + * ], + **/ + + /** + * index without no options + */ + test("normal index should store in/out direction Edges with Degrees") { + var edges = getEdgesSync(getQuery(Seq(0), "out", index1)) + (edges \ "results").as[Seq[JsValue]].size should be(3) + (edges \\ "_degree").map(_.as[Long]).sum should be(3) + + edges = getEdgesSync(getQuery(Seq(1, 2, 3), "in", index1)) + (edges \ "results").as[Seq[JsValue]].size should be(3) + (edges \\ "_degree").map(_.as[Long]).sum should be(3) + } + + /** + * { "out": {"method": "drop", "storeDegree": false } } + */ + test("storeDegree: store out direction Edge and drop Degree") { + val edges = getEdgesSync(getQuery(Seq(0), "out", idxStoreOutDropDegree)) + (edges \ "results").as[Seq[JsValue]].size should be(3) + (edges \\ "_degree").map(_.as[Long]).sum should be(0) + } + + /** + * { "in": { "method": "drop", "storeDegree": false } } + */ + test("storeDegree: store in direction Edge and drop Degree") { + val edges = getEdgesSync(getQuery(Seq(1, 2, 3), "in", idxStoreInDropDegree)) + (edges \ "results").as[Seq[JsValue]].size should be(3) + (edges \\ "_degree").map(_.as[Long]).sum should be(0) + } + + /** + * { "out": {"method": "drop", "storeDegree": false } } + */ + test("index for in direction should drop out direction edge") { + val edges = getEdgesSync(getQuery(Seq(0), "out", idxStoreIn)) + (edges \ "results").as[Seq[JsValue]].size should be(0) + (edges \\ "_degree").map(_.as[Long]).sum should be(0) + } + + test("index for in direction should store in direction edge") { + val edges = getEdgesSync(getQuery(Seq(1, 2, 3), "in", idxStoreIn)) + (edges \ "results").as[Seq[JsValue]].size should be(3) + (edges \\ "_degree").map(_.as[Long]).sum should be(3) + } + + /** + * { "in": {"method": "drop", "storeDegree": false } } + */ + test("index for out direction should store out direction edge") { + val edges = getEdgesSync(getQuery(Seq(0), "out", idxStoreOut)) + (edges \ "results").as[Seq[JsValue]].size should be(3) + (edges \\ "_degree").map(_.as[Long]).sum should be(3) + } + + test("index for out direction should drop in direction edge") { + val edges = getEdgesSync(getQuery(Seq(1, 2, 3), "in", idxStoreOut)) + (edges \ "results").as[Seq[JsValue]].size should be(0) + (edges \\ "_degree").map(_.as[Long]).sum should be(0) + } + + /** + * { "out": {"method": "drop", "storeDegree": false} } + */ + test("index for in direction should drop in direction edge and store degree") { + val edges = getEdgesSync(getQuery(Seq(1, 2, 3), "in", idxDropInStoreDegree)) + (edges \ "results").as[Seq[JsValue]].size should be(0) + (edges \\ "_degree").map(_.as[Long]).sum should be(3) + } + + /** + * { "in": {"method": "drop", "storeDegree": false }, "out": {"method": "drop"} } + */ + test("index for out direction should drop out direction edge and store degree") { + val edges = getEdgesSync(getQuery(Seq(0), "out", idxDropOutStoreDegree)) + (edges \ "results").as[Seq[JsValue]].size should be(0) + (edges \\ "_degree").map(_.as[Long]).sum should be(3) + } +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b639996b/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala index ce030bb..7c5d55d 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala @@ -20,7 +20,9 @@ package org.apache.s2graph.core.index import org.apache.s2graph.core.Integrate.IntegrateCommon -import org.apache.s2graph.core.{Management, S2Vertex} +import org.apache.s2graph.core.S2Vertex +import org.apache.tinkerpop.gremlin.process.traversal.step.util.HasContainer +import org.apache.tinkerpop.gremlin.process.traversal.P import org.apache.s2graph.core.mysqls._ import org.apache.s2graph.core.types.{InnerVal, InnerValLikeWithTs} import scala.collection.JavaConversions._ @@ -29,7 +31,10 @@ class IndexProviderTest extends IntegrateCommon { val indexProvider = IndexProvider(config) val numOfTry = 1 + lazy val gIndex = management.buildGlobalIndex("test1", Seq("_timestamp", "weight", "time")) + test("test vertex write/query") { + gIndex import TestUtil._ // Management.addVertexProp(testServiceName, testColumnName, "time", "long") @@ -59,7 +64,9 @@ class IndexProviderTest extends IntegrateCommon { indexProvider.mutateVertices(vertices) (0 until numOfTry).foreach { ith => - var ids = indexProvider.fetchVertexIds("_timestamp: 1") + val hasContainer = new HasContainer("_timestamp", P.eq(Long.box(1))) + + var ids = indexProvider.fetchVertexIds(Seq(hasContainer)) ids.head shouldBe vertex.id ids.foreach { id => @@ -67,6 +74,7 @@ class IndexProviderTest extends IntegrateCommon { } } } + test("test edge write/query ") { import TestUtil._ val testLabelName = TestUtil.testLabelName @@ -95,8 +103,9 @@ class IndexProviderTest extends IntegrateCommon { // match (0 until numOfTry).foreach { _ => - - val ids = indexProvider.fetchEdgeIds("time: 10 AND _timestamp: 1") + val hasContainers = Seq(new HasContainer("time", P.eq(Int.box(10))), + new HasContainer("_timestamp", P.eq(Int.box(1)))) + val ids = indexProvider.fetchEdgeIds(hasContainers) ids.head shouldBe edge.edgeId ids.foreach { id => @@ -106,7 +115,9 @@ class IndexProviderTest extends IntegrateCommon { // match and not (0 until numOfTry).foreach { _ => - val ids = indexProvider.fetchEdgeIds("time: 20 AND NOT _timestamp: 1") + val hasContainers = Seq(new HasContainer("time", P.eq(Int.box(20))), + new HasContainer("_timestamp", P.neq(Int.box(1)))) + val ids = indexProvider.fetchEdgeIds(hasContainers) // ids.size shouldBe 0 ids.size shouldBe numOfOthers @@ -118,7 +129,9 @@ class IndexProviderTest extends IntegrateCommon { // range (0 until numOfTry).foreach { _ => - val ids = indexProvider.fetchEdgeIds("time: [0 TO 10]") + val hasContainers = Seq(new HasContainer("time", + P.inside(Int.box(0), Int.box(11)))) + val ids = indexProvider.fetchEdgeIds(hasContainers) // ids.size shouldBe 0 ids.size shouldBe 1 @@ -128,4 +141,52 @@ class IndexProviderTest extends IntegrateCommon { } } } + + test("buildQuerySingleString") { + // (weight: 34) AND (weight: [0.5 TO *] AND price: 30) + + var hasContainer = new HasContainer("weight", P.eq(Double.box(0.5))) + + var queryString = IndexProvider.buildQuerySingleString(hasContainer) + + println(s"[[QueryString]]: ${queryString}") + + hasContainer = new HasContainer("weight", P.gte(Double.box(0.5))) + queryString = IndexProvider.buildQuerySingleString(hasContainer) + + println(s"[[QueryString]]: ${queryString}") + + hasContainer = new HasContainer("weight", P.within(Double.box(0.5), Double.box(0.7))) + queryString = IndexProvider.buildQuerySingleString(hasContainer) + + println(s"[[QueryString]]: ${queryString}") + + hasContainer = new HasContainer("weight", P.without(Double.box(0.5), Double.box(0.7))) + queryString = IndexProvider.buildQuerySingleString(hasContainer) + + println(s"[[QueryString]]: ${queryString}") + } + + test("buildQueryString") { + // (weight: 34) AND (weight: [1 TO 100]) + + var hasContainers = Seq( + new HasContainer("weight", P.eq(Int.box(34))), + new HasContainer("weight", P.between(Int.box(1), Int.box(100))) + ) + + var queryString = IndexProvider.buildQueryString(hasContainers) + println(s"[[QueryString]: ${queryString}") + + hasContainers = Seq( + new HasContainer("weight", P.eq(Int.box(34))), + new HasContainer("weight", P.outside(Int.box(1), Int.box(100))) + ) + + queryString = IndexProvider.buildQueryString(hasContainers) + println(s"[[QueryString]: ${queryString}") + } + + + } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b639996b/s2core/src/test/scala/org/apache/s2graph/core/models/GlobalIndexTest.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/models/GlobalIndexTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/models/GlobalIndexTest.scala new file mode 100644 index 0000000..18b1572 --- /dev/null +++ b/s2core/src/test/scala/org/apache/s2graph/core/models/GlobalIndexTest.scala @@ -0,0 +1,59 @@ +package org.apache.s2graph.core.models + +import org.apache.s2graph.core.TestCommonWithModels +import org.apache.s2graph.core.mysqls.GlobalIndex +import org.apache.tinkerpop.gremlin.process.traversal.P +import org.apache.tinkerpop.gremlin.process.traversal.step.util.HasContainer +import org.scalatest.{BeforeAndAfterAll, FunSuite, Matchers} +import scala.collection.JavaConversions._ + +class GlobalIndexTest extends FunSuite with Matchers with TestCommonWithModels with BeforeAndAfterAll { + override def beforeAll(): Unit = { + initTests() + } + + override def afterAll(): Unit = { + graph.shutdown() + } + + test("test buildGlobalIndex.") { + management.buildGlobalIndex("test_global", Seq("weight", "date", "name")) + } + + test("findGlobalIndex.") { + // (weight: 34) AND (weight: [1 TO 100]) + val idx1 = management.buildGlobalIndex("test-1", Seq("weight", "age", "name")) + val idx2 = management.buildGlobalIndex("test-2", Seq("gender", "age")) + val idx3 = management.buildGlobalIndex("test-3", Seq("class")) + + var hasContainers = Seq( + new HasContainer("weight", P.eq(Int.box(34))), + new HasContainer("age", P.between(Int.box(1), Int.box(100))) + ) + + GlobalIndex.findGlobalIndex(hasContainers) shouldBe(Option(idx1)) + + hasContainers = Seq( + new HasContainer("gender", P.eq(Int.box(34))), + new HasContainer("age", P.eq(Int.box(34))), + new HasContainer("class", P.eq(Int.box(34))) + ) + + GlobalIndex.findGlobalIndex(hasContainers) shouldBe(Option(idx2)) + + hasContainers = Seq( + new HasContainer("class", P.eq(Int.box(34))), + new HasContainer("_", P.eq(Int.box(34))) + ) + + GlobalIndex.findGlobalIndex(hasContainers) shouldBe(Option(idx3)) + + hasContainers = Seq( + new HasContainer("key", P.eq(Int.box(34))), + new HasContainer("value", P.eq(Int.box(34))) + ) + + GlobalIndex.findGlobalIndex(hasContainers) shouldBe(None) + + } +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b639996b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala index 1d06e46..e71ccb3 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala @@ -154,6 +154,7 @@ class S2GraphProvider extends AbstractGraphProvider { val defaultService = Service.findByName(S2Graph.DefaultServiceName).getOrElse(throw new IllegalStateException("default service is not initialized.")) val defaultServiceColumn = ServiceColumn.find(defaultService.id.get, S2Graph.DefaultColumnName).getOrElse(throw new IllegalStateException("default column is not initialized.")) + val allProps = scala.collection.mutable.Set.empty[Prop] var knowsProp = Vector( Prop("weight", "0.0", "double"), Prop("data", "-", "string"), @@ -171,8 +172,9 @@ class S2GraphProvider extends AbstractGraphProvider { Prop("data", "-", "string"), Prop("name", "-", "string") ) + allProps ++= knowsProp - // Change dataType for ColumnMeta('aKey') for PropertyFeatureSupportTest + // Change dataType for ColumnMeta('aKey') for PropertyFeatureSupportTest if (testClass.getSimpleName == "PropertyFeatureSupportTest") { knowsProp = knowsProp.filterNot(_.name == "aKey") @@ -210,28 +212,40 @@ class S2GraphProvider extends AbstractGraphProvider { } // columns - val personColumn = Management.createServiceColumn(defaultService.serviceName, "person", "integer", - Seq(Prop("name", "-", "string"), Prop("age", "0", "integer"), Prop("location", "-", "string"))) - val softwareColumn = Management.createServiceColumn(defaultService.serviceName, "software", "integer", - Seq(Prop("name", "-", "string"), Prop("lang", "-", "string"), Prop("temp", "-", "string"))) - - val productColumn = Management.createServiceColumn(defaultService.serviceName, "product", "integer", Nil) - val dogColumn = Management.createServiceColumn(defaultService.serviceName, "dog", "integer", Nil) - val animalColumn = Management.createServiceColumn(defaultService.serviceName, "animal", "integer", Seq(Prop("age", "0", "integer"), Prop("name", "-", "string"))) - val songColumn = Management.createServiceColumn(defaultService.serviceName, "song", "integer", - Seq(Prop("name", "-", "string"), Prop("songType", "-", "string"), Prop("performances", "0", "integer")) - ) - val artistColumn = Management.createServiceColumn(defaultService.serviceName, "artist", "integer", - Seq(Prop("name", "-", "string")) - ) - val stephenColumn = Management.createServiceColumn(defaultService.serviceName, "STEPHEN", "integer", - Seq(Prop("name", "-", "string")) - ) -// val vertexColumn = Management.createServiceColumn(service.serviceName, "vertex", "integer", Seq(Prop(T.id.toString, "-1", "integer"), Prop("name", "-", "string"), Prop("age", "-1", "integer"), Prop("lang", "scala", "string"))) + val personProps = Seq(Prop("name", "-", "string"), Prop("age", "0", "integer"), Prop("location", "-", "string")) + allProps ++= personProps + val personColumn = Management.createServiceColumn(defaultService.serviceName, "person", "integer", personProps) + + val softwareProps = Seq(Prop("name", "-", "string"), Prop("lang", "-", "string"), Prop("temp", "-", "string")) + allProps ++= softwareProps + val softwareColumn = Management.createServiceColumn(defaultService.serviceName, "software", "integer", softwareProps) + + val productProps = Nil + val productColumn = Management.createServiceColumn(defaultService.serviceName, "product", "integer", productProps) + + val dogProps = Nil + val dogColumn = Management.createServiceColumn(defaultService.serviceName, "dog", "integer", dogProps) + + val animalProps = Seq(Prop("age", "0", "integer"), Prop("name", "-", "string")) + allProps ++= animalProps + val animalColumn = Management.createServiceColumn(defaultService.serviceName, "animal", "integer", animalProps) + + val songProps = Seq(Prop("name", "-", "string"), Prop("songType", "-", "string"), Prop("performances", "0", "integer")) + allProps ++= songProps + val songColumn = Management.createServiceColumn(defaultService.serviceName, "song", "integer", songProps) + + val artistProps = Seq(Prop("name", "-", "string")) + allProps ++= artistProps + val artistColumn = Management.createServiceColumn(defaultService.serviceName, "artist", "integer", artistProps) + + + val stephenProps = Seq(Prop("name", "-", "string")) + allProps ++= stephenProps + val stephenColumn = Management.createServiceColumn(defaultService.serviceName, "STEPHEN", "integer", stephenProps) // labels val createdProps = Seq(Prop("weight", "0.0", "double"), Prop("name", "-", "string")) - + allProps ++= createdProps val created = if (loadGraphWith != null && loadGraphWith.value() == GraphData.MODERN) { mnt.createLabel("created", @@ -245,17 +259,21 @@ class S2GraphProvider extends AbstractGraphProvider { true, defaultService.serviceName, Nil, createdProps, "strong", None, None) } - + val boughtProps = Seq(Prop("x", "-", "string"), Prop("y", "-", "string")) + allProps ++= boughtProps val bought = mnt.createLabel("bought", defaultService.serviceName, "person", "integer", defaultService.serviceName, "product", "integer", - true, defaultService.serviceName, Nil, Seq(Prop("x", "-", "string"), Prop("y", "-", "string")), "strong", None, None, + true, defaultService.serviceName, Nil, boughtProps, "strong", None, None, options = Option("""{"skipReverse": true}""")) + val testProps = Seq(Prop("xxx", "-", "string")) + allProps ++= testProps val test = mnt.createLabel("test", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, - true, defaultService.serviceName, Nil, Seq(Prop("xxx", "-", "string")), "weak", None, None, + true, defaultService.serviceName, Nil, testProps, "weak", None, None, options = Option("""{"skipReverse": true}""")) val selfProps = Seq(Prop("__id", "-", "string"), Prop("acl", "-", "string"), Prop("test", "-", "string"), Prop("name", "-", "string"), Prop("some", "-", "string")) + allProps ++= selfProps val self = if (loadGraphWith != null && loadGraphWith.value() == GraphData.CLASSIC) { @@ -271,188 +289,230 @@ class S2GraphProvider extends AbstractGraphProvider { options = Option("""{"skipReverse": false}""")) } + val friendsProps = Seq(Prop("weight", "0.0", "double")) + allProps ++= friendsProps + val friends = if (testClass.getSimpleName == "IoVertexTest") { mnt.createLabel("friends", defaultService.serviceName, "person", "integer", defaultService.serviceName, "person", "integer", - true, defaultService.serviceName, Nil, Seq(Prop("weight", "0.0", "double")), + true, defaultService.serviceName, Nil, friendsProps, "strong", None, None, options = Option("""{"skipReverse": false}""")) } else { - mnt.createLabel("friends", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, + mnt.createLabel("friends", defaultService.serviceName, defaultServiceColumn.columnName, + defaultServiceColumn.columnType, defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, true, defaultService.serviceName, Nil, Nil, "strong", None, None, options = Option("""{"skipReverse": false}""")) } + val friendProps = Seq( + Prop("name", "-", "string"), + Prop("location", "-", "string"), + Prop("status", "-", "string"), + Prop("weight", "0.0", "double"), + Prop("acl", "-", "string") + ) + allProps ++= friendProps + val friend = if (testClass.getSimpleName == "IoEdgeTest") { mnt.createLabel("friend", defaultService.serviceName, "person", "integer", defaultService.serviceName, "person", "integer", true, defaultService.serviceName, Nil, - Seq( - Prop("name", "-", "string"), - Prop("location", "-", "string"), - Prop("status", "-", "string"), - Prop("weight", "0.0", "double"), - Prop("acl", "-", "string") - ), "strong", None, None, + friendProps, "strong", None, None, options = Option("""{"skipReverse": false}""")) } else { mnt.createLabel("friend", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, true, defaultService.serviceName, Nil, - Seq( - Prop("name", "-", "string"), - Prop("location", "-", "string"), - Prop("status", "-", "string"), - Prop("weight", "0.0", "double"), - Prop("acl", "-", "string") - ), "strong", None, None, + friendProps, "strong", None, None, options = Option("""{"skipReverse": false}""") ) } - val hate = mnt.createLabel("hate", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, - true, defaultService.serviceName, Nil, Nil, "strong", None, None, + val hateProps = Nil + val hate = mnt.createLabel("hate", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, + defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, + true, defaultService.serviceName, Nil, hateProps, "strong", None, None, options = Option("""{"skipReverse": false}""") ) + val collaboratorProps = Seq(Prop("location", "-", "string")) + allProps ++= collaboratorProps + val collaborator = mnt.createLabel("collaborator", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, true, defaultService.serviceName, Nil, - Seq( - Prop("location", "-", "string") - ), - "strong", None, None, + collaboratorProps, "strong", None, None, options = Option("""{"skipReverse": true}""") ) - val test1 = mnt.createLabel("test1", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, - true, defaultService.serviceName, Nil, Nil, "weak", None, None, + val test1Props = Nil + val test1 = mnt.createLabel("test1", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, + defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, + true, defaultService.serviceName, Nil, test1Props, "weak", None, None, options = Option("""{"skipReverse": false}""") ) - val test2 = mnt.createLabel("test2", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, - true, defaultService.serviceName, Nil, Nil, "weak", None, None, + + val test2Props = Nil + val test2 = mnt.createLabel("test2", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, + defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, + true, defaultService.serviceName, Nil, test2Props, "weak", None, None, options = Option("""{"skipReverse": false}""") ) - val test3 = mnt.createLabel("test3", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, - true, defaultService.serviceName, Nil, Nil, "weak", None, None, + + val test3Props = Nil + val test3 = mnt.createLabel("test3", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, + defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, + true, defaultService.serviceName, Nil, test3Props, "weak", None, None, options = Option("""{"skipReverse": false}""") ) - val pets = mnt.createLabel("pets", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, - true, defaultService.serviceName, Nil, Nil, "strong", None, None, + + val petsProps = Nil + val pets = mnt.createLabel("pets", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, + defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, + true, defaultService.serviceName, Nil, petsProps, "strong", None, None, options = Option("""{"skipReverse": false}""") ) - val walks = mnt.createLabel("walks", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, + + val walksProps = Seq(Prop("location", "-", "string")) + allProps ++= walksProps + + val walks = mnt.createLabel("walks", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, + defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, true, defaultService.serviceName, Nil, - Seq( - Prop("location", "-", "string") - ), "strong", None, None, + walksProps, "strong", None, None, options = Option("""{"skipReverse": false}""") ) + + val livesWithProps = Nil val livesWith = mnt.createLabel("livesWith", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, - true, defaultService.serviceName, Nil, Nil, "strong", None, None, + true, defaultService.serviceName, Nil, livesWithProps, "strong", None, None, options = Option("""{"skipReverse": false}""") ) + val hatesProps = Nil val hates = mnt.createLabel("hates", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, - true, defaultService.serviceName, Nil, Nil, "weak", None, None, + true, defaultService.serviceName, Nil, hatesProps, "weak", None, None, options = Option("""{"skipReverse": false}""") ) + val linkProps = Nil val link = mnt.createLabel("link", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, - true, defaultService.serviceName, Nil, Nil, "strong", None, None, + true, defaultService.serviceName, Nil, linkProps, "strong", None, None, options = Option("""{"skipReverse": false}""") ) + val codeveloperProps = Seq( Prop("year", "0", "integer")) + allProps ++= codeveloperProps + val codeveloper = mnt.createLabel("codeveloper", defaultService.serviceName, "person", "integer", defaultService.serviceName, "person", "integer", true, defaultService.serviceName, Nil, - Seq( - Prop("year", "0", "integer") - ), "strong", None, None, + codeveloperProps, "strong", None, None, options = Option("""{"skipReverse": false}""") ) + val createdByProps = Seq( + Prop("weight", "0.0", "double"), + Prop("year", "0", "integer"), + Prop("acl", "-", "string") + ) + allProps ++= createdByProps + val createdBy = mnt.createLabel("createdBy", defaultService.serviceName, "software", "integer", defaultService.serviceName, "person", "integer", true, defaultService.serviceName, Nil, - Seq( - Prop("weight", "0.0", "double"), - Prop("year", "0", "integer"), - Prop("acl", "-", "string") - ), "strong", None, None) + createdByProps, "strong", None, None) + + val existsWithProps = Seq(Prop("time", "-", "string")) + allProps ++= existsWithProps val existsWith = mnt.createLabel("existsWith", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, - true, defaultService.serviceName, Nil, Seq(Prop("time", "-", "string")), "strong", None, None, + true, defaultService.serviceName, Nil, existsWithProps, "strong", None, None, options = Option("""{"skipReverse": false}""") ) + val followedByProps = Seq(Prop("weight", "0.0", "double")) + allProps ++= followedByProps + val followedBy = mnt.createLabel("followedBy", defaultService.serviceName, "song", "integer", defaultService.serviceName, "song", "integer", - true, defaultService.serviceName, Nil, Seq(Prop("weight", "0.0", "double")), "strong", None, None, + true, defaultService.serviceName, Nil, followedByProps, "strong", None, None, options = Option("""{"skipReverse": false}""") ) + val writtenByProps = Seq(Prop("weight", "0.0", "double")) + allProps ++= writtenByProps + val writtenBy = mnt.createLabel("writtenBy", defaultService.serviceName, "song", "integer", defaultService.serviceName, "artist", "integer", - true, defaultService.serviceName, Nil, Seq(Prop("weight", "0.0", "double")), "strong", None, None, + true, defaultService.serviceName, Nil, writtenByProps, "strong", None, None, options = Option("""{"skipReverse": false}""") ) + val sungByProps = Seq(Prop("weight", "0.0", "double")) + allProps ++= sungByProps + val sungBy = mnt.createLabel("sungBy", defaultService.serviceName, "song", "integer", defaultService.serviceName, "artist", "integer", - true, defaultService.serviceName, Nil, Seq(Prop("weight", "0.0", "double")), "strong", None, None, + true, defaultService.serviceName, Nil, sungByProps, "strong", None, None, options = Option("""{"skipReverse": false}""") ) + val usesProps = Nil val uses = mnt.createLabel("uses", defaultService.serviceName, "person", "integer", defaultService.serviceName, "software", "integer", - true, defaultService.serviceName, Nil, Nil, "strong", None, None, + true, defaultService.serviceName, Nil, usesProps, "strong", None, None, options = Option("""{"skipReverse": false}""") ) + val likesProps = Seq(Prop("year", "0", "integer")) + allProps ++= likesProps + val likes = mnt.createLabel("likes", defaultService.serviceName, "person", "integer", defaultService.serviceName, "person", "integer", true, defaultService.serviceName, Nil, - Seq( - Prop("year", "0", "integer") - ), "strong", None, None, + likesProps, "strong", None, None, options = Option("""{"skipReverse": false}""") ) + val fooProps = Seq(Prop("year", "0", "integer")) + allProps ++= fooProps + val foo = mnt.createLabel("foo", defaultService.serviceName, "person", "integer", defaultService.serviceName, "person", "integer", true, defaultService.serviceName, Nil, - Seq( - Prop("year", "0", "integer") - ), "strong", None, None, + fooProps, "strong", None, None, options = Option("""{"skipReverse": false}""") ) + val barProps = Seq(Prop("year", "0", "integer")) + allProps ++= barProps + val bar = mnt.createLabel("bar", defaultService.serviceName, "person", "integer", defaultService.serviceName, "person", "integer", true, defaultService.serviceName, Nil, - Seq( - Prop("year", "0", "integer") - ), "strong", None, None, + barProps, "strong", None, None, options = Option("""{"skipReverse": false}""") ) + val globalIndex = mnt.buildGlobalIndex("global", allProps.map(_.name).toSeq) super.loadGraphData(graph, loadGraphWith, testClass, testName) } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b639996b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala index 3fcfb2b..3e618ea 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala @@ -24,10 +24,10 @@ import java.util.function.Predicate import org.apache.s2graph.core.GraphExceptions.LabelNotExistException import org.apache.s2graph.core.Management.JsonModel.Prop import org.apache.s2graph.core._ -import org.apache.s2graph.core.mysqls.{Service, ServiceColumn} +import org.apache.s2graph.core.mysqls.{GlobalIndex, Service, ServiceColumn} import org.apache.s2graph.core.tinkerpop.S2GraphProvider import org.apache.s2graph.core.utils.logger -import org.apache.tinkerpop.gremlin.process.traversal.Scope +import org.apache.tinkerpop.gremlin.process.traversal.{Compare, P, Scope} import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.{in, out} import org.apache.tinkerpop.gremlin.structure._ @@ -41,7 +41,7 @@ class S2GraphTest extends FunSuite with Matchers with TestCommonWithModels { initTests() val g = new S2Graph(config) - + lazy val gIndex = management.buildGlobalIndex("S2GraphTest", Seq("weight")) def printEdges(edges: Seq[Edge]): Unit = { edges.foreach { edge => logger.debug(s"[FetchedEdge]: $edge") @@ -153,33 +153,33 @@ class S2GraphTest extends FunSuite with Matchers with TestCommonWithModels { // true, service.serviceName, Nil, Seq(Prop("weight", "0.0", "float")), "strong", None, None) // - test("tinkerpop class graph test.") { - val marko = graph.addVertex(T.label, "person", T.id, Int.box(1)) - marko.property("name", "marko") - marko.property("age", Int.box(29)) - val vadas = graph.addVertex(T.label, "person", T.id, Int.box(2)) - vadas.property("name", "vadas", "age", Int.box(27)) - val lop = graph.addVertex(T.label, "software", T.id, Int.box(3), "name", "lop", "lang", "java") - val josh = graph.addVertex(T.label, "person", T.id, Int.box(4), "name", "josh", "age", Int.box(32)) - val ripple = graph.addVertex(T.label, "software", T.id, Int.box(5), "name", "ripple", "lang", "java") - val peter = graph.addVertex(T.label, "person", T.id, Int.box(6), "name", "peter", "age", Int.box(35)) - - marko.addEdge("knows", vadas, T.id, Int.box(7), "weight", Float.box(0.5f)) - marko.addEdge("knows", josh, T.id, Int.box(8), "weight", Float.box(1.0f)) - marko.addEdge("created", lop, T.id, Int.box(9), "weight", Float.box(0.4f)) - josh.addEdge("created", ripple, T.id, Int.box(10), "weight", Float.box(1.0f)) - josh.addEdge("created", lop, T.id, Int.box(11), "weight", Float.box(0.4f)) - peter.addEdge("created", lop, T.id, Int.box(12), "weight", Float.box(0.2f)) - graph.tx().commit() - - graph.traversal().V().inV() - val verticees = graph.traversal().V().asAdmin() - - - val vs = verticees.toList - - - } +// test("tinkerpop class graph test.") { +// val marko = graph.addVertex(T.label, "person", T.id, Int.box(1)) +// marko.property("name", "marko") +// marko.property("age", Int.box(29)) +// val vadas = graph.addVertex(T.label, "person", T.id, Int.box(2)) +// vadas.property("name", "vadas", "age", Int.box(27)) +// val lop = graph.addVertex(T.label, "software", T.id, Int.box(3), "name", "lop", "lang", "java") +// val josh = graph.addVertex(T.label, "person", T.id, Int.box(4), "name", "josh", "age", Int.box(32)) +// val ripple = graph.addVertex(T.label, "software", T.id, Int.box(5), "name", "ripple", "lang", "java") +// val peter = graph.addVertex(T.label, "person", T.id, Int.box(6), "name", "peter", "age", Int.box(35)) +// +// marko.addEdge("knows", vadas, T.id, Int.box(7), "weight", Float.box(0.5f)) +// marko.addEdge("knows", josh, T.id, Int.box(8), "weight", Float.box(1.0f)) +// marko.addEdge("created", lop, T.id, Int.box(9), "weight", Float.box(0.4f)) +// josh.addEdge("created", ripple, T.id, Int.box(10), "weight", Float.box(1.0f)) +// josh.addEdge("created", lop, T.id, Int.box(11), "weight", Float.box(0.4f)) +// peter.addEdge("created", lop, T.id, Int.box(12), "weight", Float.box(0.2f)) +// graph.tx().commit() +// +// graph.traversal().V().inV() +// val verticees = graph.traversal().V().asAdmin() +// +// +// val vs = verticees.toList +// +// +// } // test("addVertex with empty parameter") { // @@ -417,7 +417,10 @@ class S2GraphTest extends FunSuite with Matchers with TestCommonWithModels { //// } // } test("Modern") { + gIndex val mnt = graph.management + + S2GraphProvider.cleanupSchema S2GraphProvider.initDefaultSchema(graph) @@ -428,7 +431,7 @@ class S2GraphTest extends FunSuite with Matchers with TestCommonWithModels { val knows = mnt.createLabel("knows", S2Graph.DefaultServiceName, "person", "integer", S2Graph.DefaultServiceName, "person", "integer", - true, S2Graph.DefaultServiceName, Nil, Seq(Prop("since", "0", "integer"), Prop("year", "0", "integer")), consistencyLevel = "strong", None, None) + true, S2Graph.DefaultServiceName, Nil, Seq(Prop("weight", "0.0", "double"), Prop("year", "0", "integer")), consistencyLevel = "strong", None, None) val created = mnt.createLabel("created", S2Graph.DefaultServiceName, "person", "integer", @@ -463,7 +466,7 @@ class S2GraphTest extends FunSuite with Matchers with TestCommonWithModels { val e12 = v6.addEdge("created", v3, "weight", Double.box(0.2)) - val ls = graph.traversal().V().has("name", "josh") + val ls = graph.traversal().E().has("weight", P.eq(Double.box(0.5))) val l = ls.toList println(s"[Size]: ${l.size}")
