merge S2GRAPH-152.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/5cc41aa4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/5cc41aa4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/5cc41aa4 Branch: refs/heads/master Commit: 5cc41aa4bc66bf07f3c67813a0efb0c5c155fd50 Parents: d8de89d Author: DO YUNG YOON <steams...@apache.org> Authored: Sun Jul 30 11:07:47 2017 +0900 Committer: DO YUNG YOON <steams...@apache.org> Committed: Sun Jul 30 11:07:47 2017 +0900 ---------------------------------------------------------------------- CHANGES | 2 + dev_support/graph_mysql/schema.sql | 7 ++- .../org/apache/s2graph/core/mysqls/schema.sql | 5 +- .../org/apache/s2graph/core/Management.scala | 22 ++++---- .../s2graph/core/index/IndexProvider.scala | 8 +-- .../s2graph/core/mysqls/GlobalIndex.scala | 38 ++++++++------ .../s2graph/core/index/IndexProviderTest.scala | 2 +- .../s2graph/core/models/GlobalIndexTest.scala | 55 +++++++++++--------- .../core/tinkerpop/S2GraphProvider.scala | 7 ++- .../core/tinkerpop/structure/S2GraphTest.scala | 2 +- 10 files changed, 84 insertions(+), 64 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5cc41aa4/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 35ed7d4..4345362 100644 --- a/CHANGES +++ b/CHANGES @@ -242,6 +242,8 @@ Release 0.1.0 - unreleased S2GRAPH-131: Add actual implementation on interfaces from TinkerPop3 structure package. (Committed by DOYUNG YOON). S2GRAPH-136: Validate TinkerPop3 interface with gremlin-test suite. (Committed by DOYUNG YOON). + + S2GRAPH-152: Add buildGlobalIndex API on Management. (Committed by DOYUNG YOON). TEST http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5cc41aa4/dev_support/graph_mysql/schema.sql ---------------------------------------------------------------------- diff --git a/dev_support/graph_mysql/schema.sql b/dev_support/graph_mysql/schema.sql index 58c4328..ea72c63 100644 --- a/dev_support/graph_mysql/schema.sql +++ b/dev_support/graph_mysql/schema.sql @@ -120,22 +120,21 @@ CREATE TABLE `labels` ( ALTER TABLE labels add FOREIGN KEY(service_id) REFERENCES services(id); - -- ---------------------------- -- Table structure for `global_index` -- ---------------------------- DROP TABLE IF EXISTS `global_indices`; CREATE TABLE `global_indices` ( `id` integer NOT NULL AUTO_INCREMENT, + `element_type` varchar(64) NOT NULL, `prop_names` varchar(255) NOT NULL, `index_name` varchar(64) NOT NULL, PRIMARY KEY (`id`), - UNIQUE KEY `ux_global_index_index_name` (`index_name`), - UNIQUE KEY `ux_global_index_prop_names` (`prop_names`) + UNIQUE KEY `ux_global_index_element_type_index_name` (`element_type`, `index_name`), + UNIQUE KEY `ux_global_index_element_type_prop_names` (`element_type`, `prop_names`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -- ---------------------------- -- Table structure for `label_metas` -- ---------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5cc41aa4/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql ---------------------------------------------------------------------- diff --git a/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql b/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql index 521c9d2..f5f6a61 100644 --- a/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql +++ b/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql @@ -115,11 +115,12 @@ ALTER TABLE labels add FOREIGN KEY(service_id) REFERENCES services(id); DROP TABLE IF EXISTS `global_indices`; CREATE TABLE `global_indices` ( `id` integer NOT NULL AUTO_INCREMENT, + `element_type` varchar(64) NOT NULL, `prop_names` varchar(255) NOT NULL, `index_name` varchar(64) NOT NULL, PRIMARY KEY (`id`), - UNIQUE KEY `ux_global_index_index_name` (`index_name`), - UNIQUE KEY `ux_global_index_prop_names` (`prop_names`) + UNIQUE KEY `ux_global_index_element_type_index_name` (`element_type`, `index_name`), + UNIQUE KEY `ux_global_index_element_type_prop_names` (`element_type`, `prop_names`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5cc41aa4/s2core/src/main/scala/org/apache/s2graph/core/Management.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/Management.scala b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala index 57791af..13610b4 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/Management.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala @@ -349,15 +349,19 @@ class Management(graph: S2Graph) { old.consistencyLevel, hTableName, old.hTableTTL, old.schemaVersion, old.isAsync, old.compressionAlgorithm, old.options) } - def buildGlobalIndex(name: String, propNames: Seq[String]): GlobalIndex = { - Model.withTx { implicit session => - GlobalIndex.findBy(name, false) match { - case None => - GlobalIndex.insert(name, propNames) - GlobalIndex.findBy(name, false).get - case Some(oldIndex) => oldIndex - } - }.get + def buildGlobalVertexIndex(name: String, propNames: Seq[String]): GlobalIndex = + buildGlobalIndex(GlobalIndex.VertexType, name, propNames) + + def buildGlobalEdgeIndex(name: String, propNames: Seq[String]): GlobalIndex = + buildGlobalIndex(GlobalIndex.EdgeType, name, propNames) + + def buildGlobalIndex(elementType: String, name: String, propNames: Seq[String]): GlobalIndex = { + GlobalIndex.findBy(elementType, name, false) match { + case None => + GlobalIndex.insert(elementType, name, propNames) + GlobalIndex.findBy(elementType, name, false).get + case Some(oldIndex) => oldIndex + } } def getCurrentStorageInfo(labelName: String): Try[Map[String, String]] = for { http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5cc41aa4/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala index 57f384c..b52ffda 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala @@ -205,7 +205,7 @@ class LuceneIndexProvider(config: Config) extends IndexProvider { } override def mutateVertices(vertices: Seq[S2Vertex]): Seq[Boolean] = { - val globalIndexOptions = GlobalIndex.findAll() + val globalIndexOptions = GlobalIndex.findAll(GlobalIndex.VertexType) globalIndexOptions.map { globalIndex => val writer = getOrElseCreateIndexWriter(globalIndex.indexName) @@ -223,7 +223,7 @@ class LuceneIndexProvider(config: Config) extends IndexProvider { } override def mutateEdges(edges: Seq[S2Edge]): Seq[Boolean] = { - val globalIndexOptions = GlobalIndex.findAll() + val globalIndexOptions = GlobalIndex.findAll(GlobalIndex.EdgeType) globalIndexOptions.map { globalIndex => val writer = getOrElseCreateIndexWriter(globalIndex.indexName) @@ -244,7 +244,7 @@ class LuceneIndexProvider(config: Config) extends IndexProvider { val field = eidField val ids = new java.util.HashSet[EdgeId] - GlobalIndex.findGlobalIndex(hasContainers).map { globalIndex => + GlobalIndex.findGlobalIndex(GlobalIndex.EdgeType, hasContainers).map { globalIndex => val queryString = buildQueryString(hasContainers) try { @@ -277,7 +277,7 @@ class LuceneIndexProvider(config: Config) extends IndexProvider { val field = vidField val ids = new java.util.HashSet[VertexId] - GlobalIndex.findGlobalIndex(hasContainers).map { globalIndex => + GlobalIndex.findGlobalIndex(GlobalIndex.VertexType, hasContainers).map { globalIndex => val queryString = buildQueryString(hasContainers) try { http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5cc41aa4/s2core/src/main/scala/org/apache/s2graph/core/mysqls/GlobalIndex.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/GlobalIndex.scala b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/GlobalIndex.scala index 347c083..b3657e1 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/GlobalIndex.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/GlobalIndex.scala @@ -35,35 +35,40 @@ object GlobalIndex extends Model[GlobalIndex] { val labelField = "_label_" val serviceField = "_service_" val serviceColumnField = "_serviceColumn_" - + val EdgeType = "edge" + val VertexType = "vertex" val hiddenIndexFields = Set(vidField, eidField, labelField, serviceField, serviceColumnField) - val DefaultIndexName = GlobalIndex(None, Seq(vidField, eidField, serviceField, serviceColumnField, labelField), "_default_") val TableName = "global_indices" def apply(rs: WrappedResultSet): GlobalIndex = { - GlobalIndex(rs.intOpt("id"), rs.string("prop_names").split(",").sorted, rs.string("index_name")) + GlobalIndex(rs.intOpt("id"), + rs.string("element_type"), + rs.string("prop_names").split(",").sorted, + rs.string("index_name")) } - def findBy(indexName: String, useCache: Boolean = true)(implicit session: DBSession = AutoSession): Option[GlobalIndex] = { - val cacheKey = s"indexName=$indexName" - lazy val sql = sql"""select * from global_indices where index_name = $indexName""".map { rs => GlobalIndex(rs) }.single.apply() + def findBy(elementType: String, indexName: String, useCache: Boolean = true)(implicit session: DBSession = AutoSession): Option[GlobalIndex] = { + val cacheKey = s"elementType=$elementType:indexName=$indexName" + lazy val sql = sql"""select * from global_indices where element_type = ${elementType} and index_name = $indexName""".map { rs => GlobalIndex(rs) }.single.apply() + if (useCache) withCache(cacheKey){sql} else sql } - def insert(indexName: String, propNames: Seq[String])(implicit session: DBSession = AutoSession): Long = { + def insert(elementType: String, indexName: String, propNames: Seq[String])(implicit session: DBSession = AutoSession): Long = { val allPropNames = (hiddenIndexFields.toSeq ++ propNames).sorted - sql"""insert into global_indices(prop_names, index_name) values(${allPropNames.mkString(",")}, $indexName)""" + sql"""insert into global_indices(element_type, prop_names, index_name) + values($elementType, ${allPropNames.mkString(",")}, $indexName)""" .updateAndReturnGeneratedKey.apply() } - def findAll(useCache: Boolean = true)(implicit session: DBSession = AutoSession): Seq[GlobalIndex] = { - lazy val ls = sql"""select * from global_indices """.map { rs => GlobalIndex(rs) }.list.apply + def findAll(elementType: String, useCache: Boolean = true)(implicit session: DBSession = AutoSession): Seq[GlobalIndex] = { + lazy val ls = sql"""select * from global_indices where element_type = $elementType""".map { rs => GlobalIndex(rs) }.list.apply if (useCache) { - listCache.withCache("findAll") { + listCache.withCache(s"findAll:elementType=$elementType") { putsToCache(ls.map { globalIndex => - val cacheKey = s"indexName=${globalIndex.indexName}" + val cacheKey = s"elementType=${globalIndex.elementType}:indexName=${globalIndex.indexName}" cacheKey -> globalIndex }) ls @@ -73,9 +78,9 @@ object GlobalIndex extends Model[GlobalIndex] { } } - def findGlobalIndex(hasContainers: java.util.List[HasContainer])(implicit session: DBSession = AutoSession): Option[GlobalIndex] = { + def findGlobalIndex(elementType: String, hasContainers: java.util.List[HasContainer])(implicit session: DBSession = AutoSession): Option[GlobalIndex] = { import scala.collection.JavaConversions._ - val indices = findAll(useCache = true) + val indices = findAll(elementType, useCache = true) val keys = hasContainers.map(_.getKey) val sorted = indices.map { index => @@ -88,6 +93,9 @@ object GlobalIndex extends Model[GlobalIndex] { } -case class GlobalIndex(id: Option[Int], propNames: Seq[String], indexName: String) { +case class GlobalIndex(id: Option[Int], + elementType: String, + propNames: Seq[String], + indexName: String) { lazy val propNamesSet = propNames.toSet } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5cc41aa4/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala index c84d697..8294a8f 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala @@ -34,7 +34,7 @@ class IndexProviderTest extends IntegrateCommon { val indexProvider = IndexProvider(config) val numOfTry = 1 - lazy val gIndex = management.buildGlobalIndex("test1", Seq("_timestamp", "weight", "time")) + lazy val gIndex = management.buildGlobalIndex(GlobalIndex.EdgeType, "test1", Seq("_timestamp", "weight", "time")) // test("test vertex write/query") { // gIndex http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5cc41aa4/s2core/src/test/scala/org/apache/s2graph/core/models/GlobalIndexTest.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/models/GlobalIndexTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/models/GlobalIndexTest.scala index b779806..a732564 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/models/GlobalIndexTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/models/GlobalIndexTest.scala @@ -36,43 +36,46 @@ class GlobalIndexTest extends FunSuite with Matchers with TestCommonWithModels w } test("test buildGlobalIndex.") { - management.buildGlobalIndex("test_global", Seq("weight", "date", "name")) + Seq(GlobalIndex.EdgeType, GlobalIndex.VertexType).foreach { elementType => + management.buildGlobalIndex(elementType, "test_global", Seq("weight", "date", "name")) + } } test("findGlobalIndex.") { // (weight: 34) AND (weight: [1 TO 100]) - val idx1 = management.buildGlobalIndex("test-1", Seq("weight", "age", "name")) - val idx2 = management.buildGlobalIndex("test-2", Seq("gender", "age")) - val idx3 = management.buildGlobalIndex("test-3", Seq("class")) + Seq(GlobalIndex.EdgeType, GlobalIndex.VertexType).foreach { elementType => + val idx1 = management.buildGlobalIndex(elementType, "test-1", Seq("weight", "age", "name")) + val idx2 = management.buildGlobalIndex(elementType, "test-2", Seq("gender", "age")) + val idx3 = management.buildGlobalIndex(elementType, "test-3", Seq("class")) - var hasContainers = Seq( - new HasContainer("weight", P.eq(Int.box(34))), - new HasContainer("age", P.between(Int.box(1), Int.box(100))) - ) + var hasContainers = Seq( + new HasContainer("weight", P.eq(Int.box(34))), + new HasContainer("age", P.between(Int.box(1), Int.box(100))) + ) - GlobalIndex.findGlobalIndex(hasContainers) shouldBe(Option(idx1)) + GlobalIndex.findGlobalIndex(elementType, hasContainers) shouldBe(Option(idx1)) - hasContainers = Seq( - new HasContainer("gender", P.eq(Int.box(34))), - new HasContainer("age", P.eq(Int.box(34))), - new HasContainer("class", P.eq(Int.box(34))) - ) + hasContainers = Seq( + new HasContainer("gender", P.eq(Int.box(34))), + new HasContainer("age", P.eq(Int.box(34))), + new HasContainer("class", P.eq(Int.box(34))) + ) - GlobalIndex.findGlobalIndex(hasContainers) shouldBe(Option(idx2)) + GlobalIndex.findGlobalIndex(elementType, hasContainers) shouldBe(Option(idx2)) - hasContainers = Seq( - new HasContainer("class", P.eq(Int.box(34))), - new HasContainer("_", P.eq(Int.box(34))) - ) + hasContainers = Seq( + new HasContainer("class", P.eq(Int.box(34))), + new HasContainer("_", P.eq(Int.box(34))) + ) - GlobalIndex.findGlobalIndex(hasContainers) shouldBe(Option(idx3)) + GlobalIndex.findGlobalIndex(elementType, hasContainers) shouldBe(Option(idx3)) - hasContainers = Seq( - new HasContainer("key", P.eq(Int.box(34))), - new HasContainer("value", P.eq(Int.box(34))) - ) - - GlobalIndex.findGlobalIndex(hasContainers) shouldBe(None) + hasContainers = Seq( + new HasContainer("key", P.eq(Int.box(34))), + new HasContainer("value", P.eq(Int.box(34))) + ) + GlobalIndex.findGlobalIndex(elementType, hasContainers) shouldBe(None) + } } } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5cc41aa4/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala index d8367d7..8bcac9c 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala @@ -24,13 +24,14 @@ import org.apache.s2graph.core.GraphExceptions.LabelNotExistException import org.apache.s2graph.core.Management.JsonModel.Prop import org.apache.s2graph.core.S2Graph.{DefaultColumnName, DefaultServiceName} import org.apache.s2graph.core._ -import org.apache.s2graph.core.mysqls.{ColumnMeta, Label, Service, ServiceColumn} +import org.apache.s2graph.core.mysqls._ import org.apache.s2graph.core.types.{HBaseType, InnerVal, VertexId} import org.apache.s2graph.core.utils.logger import org.apache.tinkerpop.gremlin.LoadGraphWith.GraphData import org.apache.tinkerpop.gremlin.structure.{Element, Graph, T} import org.apache.tinkerpop.gremlin.{AbstractGraphProvider, LoadGraphWith} import java.util + import scala.collection.JavaConverters._ object S2GraphProvider { @@ -508,7 +509,9 @@ class S2GraphProvider extends AbstractGraphProvider { options = Option("""{"skipReverse": false}""") ) - val globalIndex = mnt.buildGlobalIndex("global", allProps.map(_.name).toSeq) + Seq(GlobalIndex.VertexType, GlobalIndex.EdgeType).foreach { elementType => + mnt.buildGlobalIndex(elementType, "global", allProps.map(_.name).toSeq) + } super.loadGraphData(graph, loadGraphWith, testClass, testName) } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5cc41aa4/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala index badfbfe..34769da 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala @@ -41,7 +41,7 @@ class S2GraphTest extends FunSuite with Matchers with TestCommonWithModels { initTests() val g = new S2Graph(config) - lazy val gIndex = management.buildGlobalIndex("S2GraphTest2", Seq("weight")) + lazy val gIndex = management.buildGlobalIndex(GlobalIndex.EdgeType, "S2GraphTest2", Seq("weight")) def printEdges(edges: Seq[Edge]): Unit = { edges.foreach { edge => logger.debug(s"[FetchedEdge]: $edge")