Repository: incubator-s2graph Updated Branches: refs/heads/master ce65b2712 -> 0a41ff0d2
add ServiceColumnIndex. Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/1f9693a8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/1f9693a8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/1f9693a8 Branch: refs/heads/master Commit: 1f9693a8fb68487c674d84e92aea30aecc7ece1f Parents: 26e4d43 Author: DO YUNG YOON <[email protected]> Authored: Mon Jul 10 10:42:25 2017 +0900 Committer: DO YUNG YOON <[email protected]> Committed: Mon Jul 10 10:46:46 2017 +0900 ---------------------------------------------------------------------- dev_support/graph_mysql/schema.sql | 22 +++ .../org/apache/s2graph/core/mysqls/schema.sql | 23 +++ .../apache/s2graph/core/mysqls/LabelIndex.scala | 8 +- .../core/mysqls/ServiceColumnIndex.scala | 174 +++++++++++++++++++ 4 files changed, 223 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/1f9693a8/dev_support/graph_mysql/schema.sql ---------------------------------------------------------------------- diff --git a/dev_support/graph_mysql/schema.sql b/dev_support/graph_mysql/schema.sql index df283a8..047f8b2 100644 --- a/dev_support/graph_mysql/schema.sql +++ b/dev_support/graph_mysql/schema.sql @@ -165,6 +165,28 @@ ALTER TABLE label_indices ADD FOREIGN KEY(label_id) REFERENCES labels(id) ON DEL -- ---------------------------- +-- Table structure for `service_column_indices` +-- ---------------------------- +DROP TABLE IF EXISTS `service_column_indices`; +CREATE TABLE `service_column_indices` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `service_id` int(11) NOT NULL, + `service_column_id` int(11) NOT NULL, + `name` varchar(64) NOT NULL DEFAULT '_PK', + `seq` tinyint(4) NOT NULL, + `meta_seqs` varchar(64) NOT NULL, + `options` text, + PRIMARY KEY (`id`), + UNIQUE KEY `ux_service_id_service_column_id_seq` (`service_id`,`service_column_id`,`seq`), + UNIQUE KEY `ux_service_id_service_column_id_name` (`service_id`, `service_column_id`,`name`), + UNIQUE KEY `ux_service_id_service_column_id_seqs` (`service_id`, `service_column_id`,`meta_seqs`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +ALTER TABLE service_column_indices ADD FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE; +ALTER TABLE service_column_indices ADD FOREIGN KEY(service_column_id) REFERENCES service_columns(id) ON DELETE CASCADE; + + +-- ---------------------------- -- Table structure for `experiments` -- ---------------------------- DROP TABLE IF EXISTS `experiments`; http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/1f9693a8/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql ---------------------------------------------------------------------- diff --git a/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql b/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql index b5e09c9..51efd0b 100644 --- a/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql +++ b/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql @@ -153,6 +153,29 @@ CREATE TABLE `label_indices` ( ALTER TABLE label_indices ADD FOREIGN KEY(label_id) REFERENCES labels(id) ON DELETE CASCADE; + +-- ---------------------------- +-- Table structure for `service_column_indices` +-- ---------------------------- +DROP TABLE IF EXISTS `service_column_indices`; +CREATE TABLE `service_column_indices` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `service_id` int(11) NOT NULL, + `service_column_id` int(11) NOT NULL, + `name` varchar(64) NOT NULL DEFAULT '_PK', + `seq` tinyint(4) NOT NULL, + `meta_seqs` varchar(64) NOT NULL, + `options` text, + PRIMARY KEY (`id`), + UNIQUE KEY `ux_service_id_service_column_id_seq` (`service_id`,`service_column_id`,`seq`), + UNIQUE KEY `ux_service_id_service_column_id_name` (`service_id`, `service_column_id`,`name`), + UNIQUE KEY `ux_service_id_service_column_id_seqs` (`service_id`, `service_column_id`,`meta_seqs`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +ALTER TABLE service_column_indices ADD FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE; +ALTER TABLE service_column_indices ADD FOREIGN KEY(service_column_id) REFERENCES service_columns(id) ON DELETE CASCADE; + + -- ---------------------------- -- Table structure for `experiments` -- ---------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/1f9693a8/s2core/src/main/scala/org/apache/s2graph/core/mysqls/LabelIndex.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/LabelIndex.scala b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/LabelIndex.scala index 7d4d715..e82e502 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/LabelIndex.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/LabelIndex.scala @@ -20,7 +20,7 @@ package org.apache.s2graph.core.mysqls import org.apache.s2graph.core.GraphUtil -import org.apache.s2graph.core.mysqls.LabelIndex.indexOption +import org.apache.s2graph.core.mysqls.LabelIndex.IndexOption import org.apache.s2graph.core.utils.logger import play.api.libs.json.{JsObject, JsString, Json} import scalikejdbc._ @@ -42,7 +42,7 @@ object LabelIndex extends Model[LabelIndex] { ) } - case class indexOption(dir: Byte, + case class IndexOption(dir: Byte, method: String, rate: Double, totalModular: Long, @@ -191,7 +191,7 @@ case class LabelIndex(id: Option[Int], labelId: Int, name: String, seq: Byte, me ) } - def parseOption(dir: String): Option[indexOption] = try { + def parseOption(dir: String): Option[IndexOption] = try { options.map { string => val jsObj = Json.parse(string) \ dir @@ -200,7 +200,7 @@ case class LabelIndex(id: Option[Int], labelId: Int, name: String, seq: Byte, me val totalModular = (jsObj \ "totalModular").asOpt[Long].getOrElse(100L) val storeDegree = (jsObj \ "storeDegree").asOpt[Boolean].getOrElse(true) - indexOption(GraphUtil.directions(dir).toByte, method, rate, totalModular, storeDegree) + IndexOption(GraphUtil.directions(dir).toByte, method, rate, totalModular, storeDegree) } } catch { case e: Exception => http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/1f9693a8/s2core/src/main/scala/org/apache/s2graph/core/mysqls/ServiceColumnIndex.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/ServiceColumnIndex.scala b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/ServiceColumnIndex.scala new file mode 100644 index 0000000..00204f1 --- /dev/null +++ b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/ServiceColumnIndex.scala @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.s2graph.core.mysqls + +import scalikejdbc.{AutoSession, DBSession, WrappedResultSet, _} + +object ServiceColumnIndex extends Model[ServiceColumnIndex] { + val dbTableName = "service_column_indices" + val DefaultName = "_PK" + val DefaultSeq = 1.toByte + val MaxOrderSeq = 7 + + def apply(rs: WrappedResultSet): ServiceColumnIndex = { + ServiceColumnIndex(rs.intOpt("id"), rs.int("service_id"), rs.int("service_column_id"), + rs.string("name"), + rs.byte("seq"), rs.string("meta_seqs").split(",").filter(_ != "").map(s => s.toByte).toList match { + case metaSeqsList => metaSeqsList + }, + rs.stringOpt("options") + ) + } + + def findById(id: Int)(implicit session: DBSession = AutoSession) = { + val cacheKey = "id=" + id + lazy val sql = sql"""select * from $dbTableName where id = ${id}""" + withCache(cacheKey) { + sql.map { rs => ServiceColumnIndex(rs) }.single.apply + }.get + } + + def findBySeqs(serviceId: Int, serviceColumnId: Int, seqs: List[Byte])(implicit session: DBSession = AutoSession): Option[ServiceColumnIndex] = { + val cacheKey = "serviceId=" + serviceId + ":serviceColumnId=" + serviceColumnId + ":seqs=" + seqs.mkString(",") + lazy val sql = + sql""" + select * from $dbTableName where service_id = $serviceId and service_column_id = $serviceColumnId and meta_seqs = ${seqs.mkString(",")} + """ + withCache(cacheKey) { + sql.map { rs => ServiceColumnIndex(rs) }.single.apply + } + } + + def findBySeq(serviceId: Int, + serviceColumnId: Int, + seq: Byte, + useCache: Boolean = true)(implicit session: DBSession = AutoSession) = { + val cacheKey = "serviceId=" + serviceId + ":serviceColumnId=" + serviceColumnId + ":seq=" + seq + lazy val sql = + sql""" + select * from $dbTableName where service_id = $serviceId and service_column_id = $serviceColumnId and seq = ${seq} + """ + if (useCache) { + withCache(cacheKey)(sql.map { rs => ServiceColumnIndex(rs) }.single.apply) + } else { + sql.map { rs => ServiceColumnIndex(rs) }.single.apply + } + } + + + def findAll(serviceId: Int, serviceColumnId: Int, useCache: Boolean = true)(implicit session: DBSession = AutoSession) = { + val cacheKey = s"serviceId=$serviceId:serviceColumnId=$serviceColumnId" + lazy val sql = + sql""" + select * from $dbTableName where service_id = ${serviceId} and seq > 0 order by seq ASC + """ + if (useCache) { + withCaches(cacheKey)( + sql.map { rs => ServiceColumnIndex(rs) }.list.apply + ) + } else { + sql.map { rs => LabelIndex(rs) }.list.apply + } + } + + def insert(serviceId: Int, + serviceColumnId: Int, + indexName: String, + seq: Byte, metaSeqs: List[Byte], options: Option[String])(implicit session: DBSession = AutoSession): Long = { + sql""" + insert into $dbTableName(service_id, service_column_id, name, seq, meta_seqs, options) + values (${serviceId}, ${serviceColumnId}, ${indexName}, ${seq}, ${metaSeqs.mkString(",")}, ${options}) + """ + .updateAndReturnGeneratedKey.apply() + } + + def findOrInsert(serviceId: Int, + serviceColumnId: Int, + indexName: String, + metaSeqs: List[Byte], + options: Option[String])(implicit session: DBSession = AutoSession): ServiceColumnIndex = { + findBySeqs(serviceId, serviceColumnId, metaSeqs) match { + case Some(s) => s + case None => + val orders = findAll(serviceId, serviceColumnId, false) + val seq = (orders.size + 1).toByte + assert(seq <= MaxOrderSeq) + val createdId = insert(serviceId, serviceColumnId, indexName, seq, metaSeqs, options) + val cacheKeys = toCacheKeys(createdId.toInt, serviceId, serviceColumnId, seq, metaSeqs) + + cacheKeys.foreach { key => + expireCache(key) + expireCaches(key) + } + findBySeq(serviceId, serviceColumnId, seq).get + } + } + + def toCacheKeys(id: Int, serviceId: Int, serviceColumnId: Int, seq: Byte, seqs: Seq[Byte]): Seq[String] = { + Seq(s"id=$id", + s"serviceId=$serviceId:serviceColumnId=$serviceColumnId:seq=$seq", + s"serviceId=$serviceId:serviceColumnId=$serviceColumnId:seqs=$seqs", + s"serviceId=$serviceId:serviceColumnId=$serviceColumnId") + } + + def delete(id: Int)(implicit session: DBSession = AutoSession) = { + val me = findById(id) + val seqs = me.metaSeqs.mkString(",") + val (serviceId, serviceColumnId, seq) = (me.serviceId, me.serviceColumnId, me.seq) + lazy val sql = sql"""delete from $dbTableName where id = ${id}""" + + sql.execute.apply() + + val cacheKeys = toCacheKeys(id, serviceId, serviceColumnId, seq, me.metaSeqs) + + cacheKeys.foreach { key => + expireCache(key) + expireCaches(key) + } + } + + def findAll()(implicit session: DBSession = AutoSession) = { + val ls = sql"""select * from $dbTableName""".map { rs => ServiceColumnIndex(rs) }.list.apply + val singles = ls.flatMap { x => + val cacheKeys = toCacheKeys(x.id.get, x.serviceId, x.serviceColumnId, x.seq, x.metaSeqs).dropRight(1) + cacheKeys.map { cacheKey => + cacheKey -> x + } + } + val multies = ls.groupBy(x => (x.serviceId, x.serviceColumnId)).map { case ((serviceId, serviceColumnId), ls) => + val cacheKey = s"serviceId=$serviceId:serviceColumnId=$serviceColumnId" + cacheKey -> ls + }.toList + + putsToCache(singles) + putsToCaches(multies) + + } +} + +case class ServiceColumnIndex(id: Option[Int], + serviceId: Int, + serviceColumnId: Int, + name: String, + seq: Byte, + metaSeqs: Seq[Byte], + options: Option[String]) { + +}
