Repository: incubator-s2graph Updated Branches: refs/heads/master 26e4d43cf -> ce65b2712
add buildGlobalIndex on Management. Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/37254640 Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/37254640 Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/37254640 Branch: refs/heads/master Commit: 37254640c730945aa6e26a8853c8bc9ec56b30ea Parents: 26e4d43 Author: DO YUNG YOON <steams...@apache.org> Authored: Sat Jul 29 06:36:35 2017 +0900 Committer: DO YUNG YOON <steams...@apache.org> Committed: Sat Jul 29 06:36:35 2017 +0900 ---------------------------------------------------------------------- dev_support/graph_mysql/schema.sql | 14 ++++ .../org/apache/s2graph/core/mysqls/schema.sql | 12 +++ .../org/apache/s2graph/core/Management.scala | 9 ++ .../s2graph/core/mysqls/GlobalIndex.scala | 87 ++++++++++++++++++++ .../apache/s2graph/core/mysqls/LabelIndex.scala | 16 ++-- 5 files changed, 130 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/37254640/dev_support/graph_mysql/schema.sql ---------------------------------------------------------------------- diff --git a/dev_support/graph_mysql/schema.sql b/dev_support/graph_mysql/schema.sql index df283a8..3c204a5 100644 --- a/dev_support/graph_mysql/schema.sql +++ b/dev_support/graph_mysql/schema.sql @@ -121,6 +121,20 @@ CREATE TABLE `labels` ( ALTER TABLE labels add FOREIGN KEY(service_id) REFERENCES services(id); +-- ---------------------------- +-- Table structure for `global_index` +-- ---------------------------- +DROP TABLE IF EXISTS `global_indices`; +CREATE TABLE `global_indices` ( + `id` integer NOT NULL AUTO_INCREMENT, + `prop_names` varchar(255) NOT NULL, + `index_name` varchar(64) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `ux_global_index_index_name` (`index_name`), + UNIQUE KEY `ux_global_index_prop_names` (`prop_names`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + -- ---------------------------- -- Table structure for `label_metas` http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/37254640/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql ---------------------------------------------------------------------- diff --git a/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql b/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql index b5e09c9..521c9d2 100644 --- a/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql +++ b/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql @@ -109,6 +109,18 @@ CREATE TABLE `labels` ( ALTER TABLE labels add FOREIGN KEY(service_id) REFERENCES services(id); +-- ---------------------------- +-- Table structure for `global_index` +-- ---------------------------- +DROP TABLE IF EXISTS `global_indices`; +CREATE TABLE `global_indices` ( + `id` integer NOT NULL AUTO_INCREMENT, + `prop_names` varchar(255) NOT NULL, + `index_name` varchar(64) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `ux_global_index_index_name` (`index_name`), + UNIQUE KEY `ux_global_index_prop_names` (`prop_names`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; -- ---------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/37254640/s2core/src/main/scala/org/apache/s2graph/core/Management.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/Management.scala b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala index 63a1727..dc70cb8 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/Management.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala @@ -348,6 +348,15 @@ class Management(graph: S2Graph) { old.consistencyLevel, hTableName, old.hTableTTL, old.schemaVersion, old.isAsync, old.compressionAlgorithm, old.options) } + def buildGlobalIndex(name: String, propNames: Seq[String]): GlobalIndex = { + GlobalIndex.findBy(name, false) match { + case None => + GlobalIndex.insert(name, propNames) + GlobalIndex.findBy(name, false).get + case Some(oldIndex) => oldIndex + } + } + def getCurrentStorageInfo(labelName: String): Try[Map[String, String]] = for { label <- Try(Label.findByName(labelName, useCache = false).get) } yield { http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/37254640/s2core/src/main/scala/org/apache/s2graph/core/mysqls/GlobalIndex.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/GlobalIndex.scala b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/GlobalIndex.scala new file mode 100644 index 0000000..1d1dfe2 --- /dev/null +++ b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/GlobalIndex.scala @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.s2graph.core.mysqls + +import org.apache.tinkerpop.gremlin.process.traversal.step.util.HasContainer +import scalikejdbc.{AutoSession, DBSession, WrappedResultSet} +import scalikejdbc._ + +object GlobalIndex extends Model[GlobalIndex] { + val vidField = "_vid_" + val eidField = "_eid_" + val labelField = "_label_" + val serviceField = "_service_" + val serviceColumnField = "_serviceColumn_" + + val hiddenIndexFields = Set(vidField, eidField, labelField, serviceField, serviceColumnField) + val DefaultIndexName = GlobalIndex(None, Seq(vidField, eidField, serviceField, serviceColumnField, labelField), "_default_") + + val TableName = "global_indices" + + def apply(rs: WrappedResultSet): GlobalIndex = { + GlobalIndex(rs.intOpt("id"), rs.string("prop_names").split(",").sorted, rs.string("index_name")) + } + + def findBy(indexName: String, useCache: Boolean = true)(implicit session: DBSession = AutoSession): Option[GlobalIndex] = { + val cacheKey = s"indexName=$indexName" + lazy val sql = sql"""select * from global_indices where index_name = $indexName""".map { rs => GlobalIndex(rs) }.single.apply() + if (useCache) withCache(cacheKey){sql} + else sql + } + + def insert(indexName: String, propNames: Seq[String])(implicit session: DBSession = AutoSession): Long = { + val allPropNames = (hiddenIndexFields.toSeq ++ propNames).sorted + sql"""insert into global_indices(prop_names, index_name) values(${allPropNames.mkString(",")}, $indexName)""" + .updateAndReturnGeneratedKey.apply() + } + + def findAll(useCache: Boolean = true)(implicit session: DBSession = AutoSession): Seq[GlobalIndex] = { + lazy val ls = sql"""select * from global_indices """.map { rs => GlobalIndex(rs) }.list.apply + if (useCache) { + listCache.withCache("findAll") { + putsToCache(ls.map { globalIndex => + val cacheKey = s"indexName=${globalIndex.indexName}" + cacheKey -> globalIndex + }) + ls + } + } else { + ls + } + } + + def findGlobalIndex(hasContainers: java.util.List[HasContainer])(implicit session: DBSession = AutoSession): Option[GlobalIndex] = { + import scala.collection.JavaConversions._ + val indices = findAll(useCache = true) + val keys = hasContainers.map(_.getKey) + + val sorted = indices.map { index => + val matched = keys.filter(index.propNamesSet) + index -> matched.length + }.filter(_._2 > 0).sortBy(_._2 * -1) + + sorted.headOption.map(_._1) + } + +} + +case class GlobalIndex(id: Option[Int], propNames: Seq[String], indexName: String) { + lazy val propNamesSet = propNames.toSet +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/37254640/s2core/src/main/scala/org/apache/s2graph/core/mysqls/LabelIndex.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/LabelIndex.scala b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/LabelIndex.scala index 7d4d715..93563ff 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/LabelIndex.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/LabelIndex.scala @@ -20,7 +20,7 @@ package org.apache.s2graph.core.mysqls import org.apache.s2graph.core.GraphUtil -import org.apache.s2graph.core.mysqls.LabelIndex.indexOption +import org.apache.s2graph.core.mysqls.LabelIndex.LabelIndexMutateOption import org.apache.s2graph.core.utils.logger import play.api.libs.json.{JsObject, JsString, Json} import scalikejdbc._ @@ -42,11 +42,11 @@ object LabelIndex extends Model[LabelIndex] { ) } - case class indexOption(dir: Byte, - method: String, - rate: Double, - totalModular: Long, - storeDegree: Boolean) { + case class LabelIndexMutateOption(dir: Byte, + method: String, + rate: Double, + totalModular: Long, + storeDegree: Boolean) { val isBufferIncrement = method == "drop" || method == "sample" || method == "hash_sample" @@ -191,7 +191,7 @@ case class LabelIndex(id: Option[Int], labelId: Int, name: String, seq: Byte, me ) } - def parseOption(dir: String): Option[indexOption] = try { + def parseOption(dir: String): Option[LabelIndexMutateOption] = try { options.map { string => val jsObj = Json.parse(string) \ dir @@ -200,7 +200,7 @@ case class LabelIndex(id: Option[Int], labelId: Int, name: String, seq: Byte, me val totalModular = (jsObj \ "totalModular").asOpt[Long].getOrElse(100L) val storeDegree = (jsObj \ "storeDegree").asOpt[Boolean].getOrElse(true) - indexOption(GraphUtil.directions(dir).toByte, method, rate, totalModular, storeDegree) + LabelIndexMutateOption(GraphUtil.directions(dir).toByte, method, rate, totalModular, storeDegree) } } catch { case e: Exception =>
