add schema
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/60c935ee Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/60c935ee Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/60c935ee Branch: refs/heads/master Commit: 60c935ee2e8d145af84a632f942cc1a7a75ca640 Parents: 54c56c3 Author: daewon <[email protected]> Authored: Fri May 4 17:18:23 2018 +0900 Committer: daewon <[email protected]> Committed: Fri May 4 17:18:23 2018 +0900 ---------------------------------------------------------------------- .../movielens/schema/edge.similar.movie.graphql | 61 ++++++++++++++++++++ .../apache/s2graph/core/model/Importer.scala | 10 +++- .../s2graph/core/model/ModelManager.scala | 9 ++- .../model/fasttext/FastTextFetcherTest.scala | 23 +++++--- .../apache/s2graph/graphql/GraphQLServer.scala | 17 ++++-- 5 files changed, 98 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/60c935ee/example/movielens/schema/edge.similar.movie.graphql ---------------------------------------------------------------------- diff --git a/example/movielens/schema/edge.similar.movie.graphql b/example/movielens/schema/edge.similar.movie.graphql new file mode 100644 index 0000000..f8ac33b --- /dev/null +++ b/example/movielens/schema/edge.similar.movie.graphql @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +mutation{ + Management{ + createLabel( + name:"similar_movie" + sourceService: { + movielens: { + columnName: Movie + } + } + targetService: { + movielens: { + columnName: Movie + } + } + serviceName: movielens + consistencyLevel: strong + props:[ + { + name: "score" + dataType: double + defaultValue: "0.0" + storeInGlobalIndex: false + } + ] + indices:{ + name:"_PK" + propNames:["score"] + } + ) { + isSuccess + message + object{ + id + name + props{ + name + } + } + } + } +} + http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/60c935ee/s2core/src/main/scala/org/apache/s2graph/core/model/Importer.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/model/Importer.scala b/s2core/src/main/scala/org/apache/s2graph/core/model/Importer.scala index 5265483..0f5bca0 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/model/Importer.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/model/Importer.scala @@ -32,6 +32,7 @@ object Importer { trait Importer { @volatile var isFinished: Boolean = false + def run(config: Config)(implicit ec: ExecutionContext): Future[Importer] def status: Boolean = isFinished @@ -40,11 +41,10 @@ trait Importer { this.isFinished = otherStatus this.isFinished } -// def status: ImportStatus -// def getImportedStorage(graphExecutionContext: ExecutionContext): Storage[_, _] def close(): Unit } + case class IdentityImporter(graph: S2GraphLike) extends Importer { override def run(config: Config)(implicit ec: ExecutionContext): Future[Importer] = { Future.successful(this) @@ -52,8 +52,11 @@ case class IdentityImporter(graph: S2GraphLike) extends Importer { override def close(): Unit = {} } + object HDFSImporter { + import scala.collection.JavaConverters._ + val PathsKey = "paths" val HDFSConfDirKey = "hdfsConfDir" @@ -66,6 +69,7 @@ object HDFSImporter { }.toMap } } + case class HDFSImporter(graph: S2GraphLike) extends Importer { import HDFSImporter._ @@ -93,7 +97,7 @@ case class HDFSImporter(graph: S2GraphLike) extends Importer { } } -// override def status: ImportStatus = ??? + // override def status: ImportStatus = ??? override def close(): Unit = {} } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/60c935ee/s2core/src/main/scala/org/apache/s2graph/core/model/ModelManager.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/model/ModelManager.scala b/s2core/src/main/scala/org/apache/s2graph/core/model/ModelManager.scala index 4afd3e3..97dd591 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/model/ModelManager.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/model/ModelManager.scala @@ -13,6 +13,7 @@ object ModelManager { } class ModelManager(s2GraphLike: S2GraphLike) { + import ModelManager._ private val fetcherPool = scala.collection.mutable.Map.empty[String, Fetcher] @@ -37,7 +38,7 @@ class ModelManager(s2GraphLike: S2GraphLike) { val className = config.getString(FetcherClassNameKey) val fetcher = Class.forName(className) - .getConstructor(classOf[S2GraphLike]) + .getConstructor(classOf[S2GraphLike]) .newInstance(s2GraphLike) .asInstanceOf[Fetcher] @@ -58,7 +59,7 @@ class ModelManager(s2GraphLike: S2GraphLike) { initFetcher(config.getConfig("fetcher")).map { fetcher => importer.setStatus(true) - + Label.updateOption(label.label, "") fetcherPool .remove(k) @@ -68,15 +69,13 @@ class ModelManager(s2GraphLike: S2GraphLike) { } fetcherPool += (k -> fetcher) - true } - - true } .onComplete { _ => logger.info(s"ImportLock release: $k") ImportLock.remove(k) } + importer } }) http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/60c935ee/s2core/src/test/scala/org/apache/s2graph/core/model/fasttext/FastTextFetcherTest.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/model/fasttext/FastTextFetcherTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/model/fasttext/FastTextFetcherTest.scala index f91e0d5..7077e52 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/model/fasttext/FastTextFetcherTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/model/fasttext/FastTextFetcherTest.scala @@ -14,23 +14,26 @@ class FastTextFetcherTest extends IntegrateCommon { import TestUtil._ test("FastTextFetcher init test.") { - val modelPath = "/Users/shon/Downloads/emoji-context-by-story-comments-20170901-20180410" + val modelPath = "./emoji" val config = ConfigFactory.parseMap(Map(FastText.DBPathKey -> modelPath).asJava) val fetcher = new FastTextFetcher(graph) Await.ready(fetcher.init(config)(ExecutionContext.Implicits.global), Duration("3 minutes")) val service = management.createService("s2graph", "localhost", "s2graph_htable", -1, None).get - val serviceColumn = - management.createServiceColumn("s2graph", "keyword", "string", Seq(Prop("age", "0", "int", true))) + val emojiColumn = + management.createServiceColumn("s2graph", "emoji", "string", Seq(Prop("url", "", "string", false))) - val labelName = "fasttext_test_label" + val sentenceColumn = + management.createServiceColumn("s2graph", "sentence", "string", Nil) + + val labelName = "sentence_emoji" Label.findByName(labelName, useCache = false).foreach { label => Label.delete(label.id.get) } val label = management.createLabel( labelName, - serviceColumn, - serviceColumn, + sentenceColumn , + emojiColumn, true, service.serviceName, Seq.empty[Index].asJava, @@ -42,18 +45,22 @@ class FastTextFetcherTest extends IntegrateCommon { "gz", "" ) - val vertex = graph.elementBuilder.toVertex(service.serviceName, serviceColumn.columnName, "ìë íì¸ì") + val vertex = graph.elementBuilder.toVertex(service.serviceName, sentenceColumn.columnName, "íë¬ì´") val queryParam = QueryParam(labelName = labelName, limit = 5) val query = Query.toQuery(srcVertices = Seq(vertex), queryParams = Seq(queryParam)) val queryRequests = Seq( QueryRequest(query, 0, vertex, queryParam) ) + val future = fetcher.fetches(queryRequests, Map.empty) val results = Await.result(future, Duration("10 seconds")) results.foreach { stepResult => stepResult.edgeWithScores.foreach { es => - println(es.edge.tgtVertex.innerIdVal) + val Array(itemId, resourceId) = es.edge.tgtVertex.innerIdVal.toString.replace("__label__", "").split("_") + val text = String.format("http://item.kakaocdn.net/dw/%s.thum_%03d.png";, itemId, Int.box(resourceId.toInt)) + + println(text) } } } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/60c935ee/s2graphql/src/main/scala/org/apache/s2graph/graphql/GraphQLServer.scala ---------------------------------------------------------------------- diff --git a/s2graphql/src/main/scala/org/apache/s2graph/graphql/GraphQLServer.scala b/s2graphql/src/main/scala/org/apache/s2graph/graphql/GraphQLServer.scala index 22ef43d..0bf62d9 100644 --- a/s2graphql/src/main/scala/org/apache/s2graph/graphql/GraphQLServer.scala +++ b/s2graphql/src/main/scala/org/apache/s2graph/graphql/GraphQLServer.scala @@ -37,7 +37,7 @@ import sangria.execution.deferred.DeferredResolver import sangria.marshalling.sprayJson._ import sangria.parser.QueryParser import sangria.schema.Schema -import spray.json.{JsObject, JsString} +import spray.json.{JsBoolean, JsObject, JsString} import scala.collection.JavaConverters._ import scala.concurrent.ExecutionContext @@ -64,12 +64,17 @@ object GraphQLServer { val schemaCache = new SafeUpdateCache(schemaConfig) def importModel(requestJSON: spray.json.JsValue)(implicit e: ExecutionContext): Route = { - val spray.json.JsObject(fields) = requestJSON - val spray.json.JsString(labelName) = fields("label") - val jsOptions = fields("options") + val ret = Try { + val spray.json.JsObject(fields) = requestJSON + val spray.json.JsString(labelName) = fields("label") + val jsOptions = fields("options") + + s2graph.management.importModel(labelName, jsOptions.compactPrint) + } - complete { - s2graph.management.importModel(labelName, jsOptions.compactPrint).map(a => OK -> JsString("")) + ret match { + case Success(f) => complete(f.map(i => OK -> JsString("start"))) + case Failure(e) => complete(InternalServerError -> spray.json.JsObject("message" -> JsString(e.toString))) } }
![http://item.kakaocdn.net/dw/%s.thum_%03d.png"](http://item.kakaocdn.net/dw/%s.thum_%03d.png"){kind=link}