Merge branch 'master' into S2GRAPH-211 * master: [S2GRAPH-210]: Rename package `mysqls` to `schema` update SafeUpdateCache constructor. fix compile error while test:compile. - rename mysqls package to schema. - remove GlobalIndex. - add toBytes, fromBytes on Schema.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/501716c8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/501716c8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/501716c8 Branch: refs/heads/master Commit: 501716c8d793590f0205e84445273b3ef7420c74 Parents: 204efab e674a25 Author: daewon <[email protected]> Authored: Fri Apr 27 16:29:58 2018 +0900 Committer: daewon <[email protected]> Committed: Fri Apr 27 16:29:58 2018 +0900 ---------------------------------------------------------------------- CHANGES | 1 + .../loader/subscriber/TransferToHFile.scala | 2 +- .../loader/subscriber/TransferToHFileTest.scala | 2 +- project/Common.scala | 2 + s2core/build.sbt | 6 +- .../core/io/tinkerpop/optimize/S2GraphStep.java | 9 - .../org/apache/s2graph/core/mysqls/schema.sql | 247 --------- .../org/apache/s2graph/core/mysqls/setup.sql | 28 - .../org/apache/s2graph/core/schema/schema.sql | 247 +++++++++ .../org/apache/s2graph/core/schema/setup.sql | 28 + .../apache/s2graph/core/ExceptionHandler.scala | 27 +- .../s2graph/core/GraphElementBuilder.scala | 2 +- .../org/apache/s2graph/core/JSONParser.scala | 2 +- .../org/apache/s2graph/core/Management.scala | 26 +- .../org/apache/s2graph/core/PostProcess.scala | 2 +- .../org/apache/s2graph/core/QueryParam.scala | 2 +- .../org/apache/s2graph/core/QueryResult.scala | 2 +- .../scala/org/apache/s2graph/core/S2Edge.scala | 2 +- .../org/apache/s2graph/core/S2EdgeBuilder.scala | 2 +- .../org/apache/s2graph/core/S2EdgeLike.scala | 2 +- .../s2graph/core/S2EdgePropertyHelper.scala | 2 +- .../scala/org/apache/s2graph/core/S2Graph.scala | 8 +- .../apache/s2graph/core/S2GraphFactory.scala | 2 +- .../org/apache/s2graph/core/S2GraphLike.scala | 2 +- .../org/apache/s2graph/core/S2Property.scala | 2 +- .../org/apache/s2graph/core/S2Vertex.scala | 2 +- .../apache/s2graph/core/S2VertexBuilder.scala | 2 +- .../org/apache/s2graph/core/S2VertexLike.scala | 2 +- .../apache/s2graph/core/S2VertexProperty.scala | 2 +- .../s2graph/core/S2VertexPropertyHelper.scala | 2 +- .../apache/s2graph/core/TraversalHelper.scala | 2 +- .../s2graph/core/index/ESIndexProvider.scala | 21 +- .../s2graph/core/index/IndexProvider.scala | 19 +- .../core/index/LuceneIndexProvider.scala | 13 +- .../apache/s2graph/core/io/Conversions.scala | 2 +- .../org/apache/s2graph/core/mysqls/Bucket.scala | 101 ---- .../apache/s2graph/core/mysqls/ColumnMeta.scala | 169 ------ .../apache/s2graph/core/mysqls/Experiment.scala | 108 ---- .../s2graph/core/mysqls/GlobalIndex.scala | 100 ---- .../org/apache/s2graph/core/mysqls/Label.scala | 511 ------------------- .../apache/s2graph/core/mysqls/LabelIndex.scala | 251 --------- .../apache/s2graph/core/mysqls/LabelMeta.scala | 221 -------- .../org/apache/s2graph/core/mysqls/Model.scala | 232 --------- .../apache/s2graph/core/mysqls/Service.scala | 128 ----- .../s2graph/core/mysqls/ServiceColumn.scala | 153 ------ .../core/mysqls/ServiceColumnIndex.scala | 174 ------- .../s2graph/core/parsers/WhereParser.scala | 2 +- .../s2graph/core/rest/RequestParser.scala | 2 +- .../apache/s2graph/core/rest/RestHandler.scala | 2 +- .../org/apache/s2graph/core/schema/Bucket.scala | 138 +++++ .../apache/s2graph/core/schema/ColumnMeta.scala | 165 ++++++ .../apache/s2graph/core/schema/Experiment.scala | 111 ++++ .../org/apache/s2graph/core/schema/Label.scala | 506 ++++++++++++++++++ .../apache/s2graph/core/schema/LabelIndex.scala | 214 ++++++++ .../apache/s2graph/core/schema/LabelMeta.scala | 217 ++++++++ .../org/apache/s2graph/core/schema/Schema.scala | 224 ++++++++ .../apache/s2graph/core/schema/Service.scala | 133 +++++ .../s2graph/core/schema/ServiceColumn.scala | 151 ++++++ .../apache/s2graph/core/storage/StorageIO.scala | 2 +- .../hbase/AsynchbaseStorageReadable.scala | 2 +- .../storage/rocks/RocksStorageReadable.scala | 2 +- .../core/storage/serde/MutationHelper.scala | 2 +- .../storage/serde/StorageDeserializable.scala | 2 +- .../storage/serde/StorageSerializable.scala | 2 +- .../tall/IndexEdgeDeserializable.scala | 2 +- .../indexedge/tall/IndexEdgeSerializable.scala | 2 +- .../wide/IndexEdgeDeserializable.scala | 2 +- .../indexedge/wide/IndexEdgeSerializable.scala | 2 +- .../tall/SnapshotEdgeDeserializable.scala | 2 +- .../tall/SnapshotEdgeSerializable.scala | 2 +- .../wide/SnapshotEdgeDeserializable.scala | 2 +- .../wide/SnapshotEdgeSerializable.scala | 2 +- .../vertex/tall/VertexDeserializable.scala | 2 +- .../vertex/wide/VertexDeserializable.scala | 2 +- .../apache/s2graph/core/types/HBaseType.scala | 2 +- .../apache/s2graph/core/types/VertexId.scala | 2 +- .../org/apache/s2graph/core/utils/Logger.scala | 10 +- .../s2graph/core/utils/SafeUpdateCache.scala | 139 ++++- .../s2graph/core/Integrate/CrudTest.scala | 2 +- .../core/Integrate/IntegrateCommon.scala | 2 +- .../apache/s2graph/core/ManagementTest.scala | 2 +- .../org/apache/s2graph/core/S2EdgeTest.scala | 2 +- .../org/apache/s2graph/core/TestCommon.scala | 2 +- .../s2graph/core/TestCommonWithModels.scala | 2 +- .../s2graph/core/benchmark/GraphUtilSpec.scala | 2 +- .../s2graph/core/index/IndexProviderTest.scala | 7 +- .../apache/s2graph/core/io/ConversionTest.scala | 2 +- .../apache/s2graph/core/models/ModelTest.scala | 59 --- .../s2graph/core/parsers/WhereParserTest.scala | 2 +- .../apache/s2graph/core/schema/SchemaTest.scala | 77 +++ .../s2graph/core/storage/StorageIOTest.scala | 2 +- .../core/storage/hbase/IndexEdgeTest.scala | 2 +- .../core/storage/rocks/RocksStorageTest.scala | 2 +- .../core/tinkerpop/S2GraphProvider.scala | 2 +- .../core/tinkerpop/structure/S2GraphTest.scala | 2 +- .../counter/core/v2/ExactStorageGraph.scala | 2 +- .../counter/core/v2/RankingStorageGraph.scala | 2 +- .../s2graph/counter/helper/CounterAdmin.scala | 2 +- .../counter/core/RankingCounterSpec.scala | 2 +- .../counter/models/CounterModelSpec.scala | 69 --- .../counter/models/CounterSchemaSpec.scala | 69 +++ .../counter/loader/core/DimensionProps.scala | 2 +- .../loader/core/CounterEtlFunctionsSpec.scala | 2 +- .../apache/s2graph/graphql/GraphQLServer.scala | 12 +- .../s2graph/graphql/bind/Unmarshaller.scala | 4 +- .../graphql/repository/GraphRepository.scala | 2 +- .../s2graph/graphql/types/FieldResolver.scala | 5 +- .../s2graph/graphql/types/ManagementType.scala | 2 +- .../apache/s2graph/graphql/types/S2Type.scala | 2 +- .../s2graph/graphql/types/StaticType.scala | 2 +- .../org/apache/s2graph/graphql/TestGraph.scala | 6 +- .../org/apache/s2graph/s2jobs/DegreeKey.scala | 2 +- .../apache/s2graph/s2jobs/S2GraphHelper.scala | 1 + .../apache/s2graph/s2jobs/BaseSparkTest.scala | 6 +- .../apache/s2graph/s2jobs/task/SinkTest.scala | 1 - .../org/apache/s2graph/rest/netty/Server.scala | 2 +- .../rest/play/controllers/AdminController.scala | 2 +- .../play/controllers/CounterController.scala | 2 +- .../rest/play/controllers/EdgeController.scala | 2 +- 119 files changed, 2578 insertions(+), 2718 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/501716c8/s2core/src/main/scala/org/apache/s2graph/core/schema/Schema.scala ---------------------------------------------------------------------- diff --cc s2core/src/main/scala/org/apache/s2graph/core/schema/Schema.scala index 0000000,ebae966..c28df80 mode 000000,100644..100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/schema/Schema.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/schema/Schema.scala @@@ -1,0 -1,223 +1,224 @@@ + /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.s2graph.core.schema + + import java.util.concurrent.Executors + import java.util.concurrent.atomic.AtomicLong + + import com.typesafe.config.{Config, ConfigFactory} + import org.apache.s2graph.core.JSONParser + import org.apache.s2graph.core.utils.{SafeUpdateCache, logger} + import play.api.libs.json.{JsObject, JsValue, Json} + import scalikejdbc._ + + import scala.concurrent.ExecutionContext + import scala.io.Source + import scala.language.{higherKinds, implicitConversions} + import scala.util.{Failure, Success, Try} + + object Schema { + var maxSize = 10000 + var ttl = 60 + var safeUpdateCache: SafeUpdateCache = _ + + val numOfThread = Runtime.getRuntime.availableProcessors() + val threadPool = Executors.newFixedThreadPool(numOfThread) + val ec = ExecutionContext.fromExecutor(threadPool) + + private val ModelReferenceCount = new AtomicLong(0L) + + def apply(config: Config) = { + Class.forName(config.getString("db.default.driver")) + + val settings = ConnectionPoolSettings( + initialSize = 10, + maxSize = 10, + connectionTimeoutMillis = 30000L, + validationQuery = "select 1;") + + ConnectionPool.singleton( + config.getString("db.default.url"), + config.getString("db.default.user"), + config.getString("db.default.password"), + settings) + + checkSchema() + safeUpdateCache = new SafeUpdateCache(config)(ec) + ModelReferenceCount.incrementAndGet() + } + + def checkSchema(): Unit = { + withTx { implicit session => + sql"""show tables""".map(rs => rs.string(1)).list.apply() + } match { + case Success(tables) => + if (tables.isEmpty) { + // this is a very simple migration tool that only supports creating + // appropriate tables when there are no tables in the database at all. + // Ideally, it should be improved to a sophisticated migration tool + // that supports versioning, etc. + logger.info("Creating tables ...") + val schema = getClass.getResourceAsStream("schema.sql") + val lines = Source.fromInputStream(schema, "UTF-8").getLines + val sources = lines.map(_.split("-- ").head.trim).mkString("\n") + val statements = sources.split(";\n") + withTx { implicit session => + statements.foreach(sql => session.execute(sql)) + } match { + case Success(_) => + logger.info("Successfully imported schema") + case Failure(e) => + throw new RuntimeException("Error while importing schema", e) + } + } + case Failure(e) => + throw new RuntimeException("Could not list tables in the database", e) + } + } + + def withTx[T](block: DBSession => T): Try[T] = { + using(DB(ConnectionPool.borrow())) { conn => + Try { + conn.begin() + val session = conn.withinTxSession() + val result = block(session) + + conn.commit() + + result + } recoverWith { + case e: Exception => + conn.rollbackIfActive() + Failure(e) + } + } + } + + def shutdown(modelDataDelete: Boolean = false) = + if (ModelReferenceCount.decrementAndGet() <= 0) { + // FIXME: When Model is served by embedded database and deleteData is set, Model deletes + // the underlying database. Its purpose is clearing runtime footprint when running tests. + if (modelDataDelete) { + withTx { implicit session => + sql"SHOW TABLES" + .map(rs => rs.string(1)) + .list + .apply() + .map { table => s"TRUNCATE TABLE $table" } + } match { + case Success(stmts) => + val newStmts = List("SET FOREIGN_KEY_CHECKS = 0") ++ stmts ++ List("SET FOREIGN_KEY_CHECKS = 1") + withTx { implicit session => + newStmts.foreach { stmt => + session.execute(stmt) + } + } match { + case Success(_) => + logger.info(s"Success to truncate models: $stmts") + case Failure(e) => + throw new IllegalStateException(s"Failed to truncate models", e) + } + case Failure(e) => + throw new IllegalStateException(s"Failed to list models", e) + } + } + // clearCache() + safeUpdateCache.shutdown() + ConnectionPool.closeAll() + } + + def loadCache() = { + Service.findAll() + ServiceColumn.findAll() + Label.findAll() + LabelMeta.findAll() + LabelIndex.findAll() + ColumnMeta.findAll() + } + + // def clearCache() = { + // Service.expireAll() + // ServiceColumn.expireAll() + // Label.expireAll() + // LabelMeta.expireAll() + // LabelIndex.expireAll() + // ColumnMeta.expireAll() + // } + + def extraOptions(options: Option[String]): Map[String, JsValue] = options match { + case None => Map.empty ++ case Some(v) if v.trim == "" => Map.empty + case Some(v) => + try { + Json.parse(v).asOpt[JsObject].map { obj => obj.fields.toMap }.getOrElse(Map.empty) + } catch { + case e: Exception => - logger.error(s"An error occurs while parsing the extra label option", e) ++ logger.error(s"An error occurs while parsing the extra label option: ${v}", e) + Map.empty + } + } + + def toStorageConfig(options: Map[String, JsValue]): Option[Config] = { + try { + options.get("storage").map { jsValue => + import scala.collection.JavaConverters._ + val configMap = jsValue.as[JsObject].fieldSet.toMap.map { case (key, value) => + key -> JSONParser.jsValueToAny(value).getOrElse(throw new RuntimeException("!!")) + } + + ConfigFactory.parseMap(configMap.asJava) + } + } catch { + case e: Exception => + logger.error(s"toStorageConfig error. use default storage", e) + None + } + } + + private def toMultiKey(key: String): String = key + ".__m__" + + def withCache[T <: AnyRef](key: String, broadcast: Boolean = true)(op: => T) = safeUpdateCache.withCache(key, broadcast)(op) + + def withCaches[T <: AnyRef](key: String, broadcast: Boolean = true)(op: => T) = safeUpdateCache.withCache(toMultiKey(key), broadcast)(op) + + def expireCache(key: String) = safeUpdateCache.invalidate(key) + + def expireCaches(key: String) = safeUpdateCache.invalidate(toMultiKey(key)) + + def putsToCacheOption[T <: AnyRef](kvs: List[(String, T)]) = kvs.foreach { + case (key, value) => safeUpdateCache.put(key, Option(value)) + } + + def putsToCaches[T <: AnyRef](kvs: List[(String, T)]) = kvs.foreach { + case (key, values) => safeUpdateCache.put(toMultiKey(key), values) + } + + def getCacheSize(): Int = safeUpdateCache.asMap().size() + + def getAllCacheData[T <: AnyRef](): (List[(String, T)], List[(String, List[T])]) = { + (Nil, Nil) + } + + def toBytes(): Array[Byte] = safeUpdateCache.toBytes() + + def fromBytes(config: Config, bytes: Array[Byte])(implicit ec: ExecutionContext): SafeUpdateCache = + SafeUpdateCache.fromBytes(config, bytes) + } + http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/501716c8/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/BaseSparkTest.scala ---------------------------------------------------------------------- diff --cc s2jobs/src/test/scala/org/apache/s2graph/s2jobs/BaseSparkTest.scala index 9037aba,e8a3e86..8dfbe1e --- a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/BaseSparkTest.scala +++ b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/BaseSparkTest.scala @@@ -20,14 -20,12 +20,12 @@@ package org.apache.s2graph.s2jobs import java.io.{File, PrintWriter} - import java.nio.file.Path import com.holdenkarau.spark.testing.DataFrameSuiteBase - import com.typesafe.config.ConfigFactory import org.apache.s2graph.core.Management.JsonModel.{Index, Prop} - import org.apache.s2graph.core.mysqls.{Label, Service, ServiceColumn} - import org.apache.s2graph.core.types.HBaseType -import org.apache.s2graph.core.schema.{Label, ServiceColumn} ++import org.apache.s2graph.core.schema.{Label, Service, ServiceColumn} import org.apache.s2graph.core.{Management, S2Graph} + import org.apache.s2graph.core.types.HBaseType import org.apache.s2graph.s2jobs.loader.GraphFileOptions import org.scalatest.{BeforeAndAfterAll, FunSuite, Matchers} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/501716c8/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/task/SinkTest.scala ----------------------------------------------------------------------
