add config 's2graph.read-only.url' for counter
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/b159a7fb Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/b159a7fb Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/b159a7fb Branch: refs/heads/feature/test_daewon Commit: b159a7fbf5f39b335c7709e0eb64f23a6cbfda7f Parents: 5acead8 Author: Jaesang Kim <[email protected]> Authored: Mon Dec 21 14:21:05 2015 +0900 Committer: Jaesang Kim <[email protected]> Committed: Mon Dec 21 14:21:05 2015 +0900 ---------------------------------------------------------------------- .../main/scala/s2/config/S2CounterConfig.scala | 5 ++-- .../s2/counter/core/v2/ExactStorageGraph.scala | 25 +++----------------- .../counter/core/v2/RankingStorageGraph.scala | 5 ++-- .../main/scala/s2/config/StreamingConfig.scala | 1 + .../src/test/resources/application.conf | 1 + 5 files changed, 11 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b159a7fb/s2counter_core/src/main/scala/s2/config/S2CounterConfig.scala ---------------------------------------------------------------------- diff --git a/s2counter_core/src/main/scala/s2/config/S2CounterConfig.scala b/s2counter_core/src/main/scala/s2/config/S2CounterConfig.scala index d0db73c..fcd0e6a 100644 --- a/s2counter_core/src/main/scala/s2/config/S2CounterConfig.scala +++ b/s2counter_core/src/main/scala/s2/config/S2CounterConfig.scala @@ -28,7 +28,7 @@ class S2CounterConfig(config: Config) extends ConfigFunctions(config) { lazy val DB_DEFAULT_USER = getOrElse("db.default.user", "graph") lazy val DB_DEFAULT_PASSWORD = getOrElse("db.default.password", "graph") - // REDIS + // Redis lazy val REDIS_INSTANCES = (for { s <- config.getStringList("redis.instances") } yield { @@ -36,8 +36,9 @@ class S2CounterConfig(config: Config) extends ConfigFunctions(config) { (sp(0), if (sp.length > 1) sp(1).toInt else 6379) }).toList - // graph + // Graph lazy val GRAPH_URL = getOrElse("s2graph.url", "http://localhost:9000";) + lazy val GRAPH_READONLY_URL = getOrElse("s2graph.read-only.url", GRAPH_URL) // Cache lazy val CACHE_TTL_SECONDS = getOrElse("cache.ttl.seconds", 600) http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b159a7fb/s2counter_core/src/main/scala/s2/counter/core/v2/ExactStorageGraph.scala ---------------------------------------------------------------------- diff --git a/s2counter_core/src/main/scala/s2/counter/core/v2/ExactStorageGraph.scala b/s2counter_core/src/main/scala/s2/counter/core/v2/ExactStorageGraph.scala index add7c61..7dcff6c 100644 --- a/s2counter_core/src/main/scala/s2/counter/core/v2/ExactStorageGraph.scala +++ b/s2counter_core/src/main/scala/s2/counter/core/v2/ExactStorageGraph.scala @@ -35,6 +35,7 @@ case class ExactStorageGraph(config: Config) extends ExactStorage { private val labelPostfix = "_counts" val s2graphUrl = s2config.GRAPH_URL + val s2graphReadOnlyUrl = s2config.GRAPH_READONLY_URL val graphOp = new GraphOperation(config) import ExactStorageGraph._ @@ -74,26 +75,6 @@ case class ExactStorageGraph(config: Config) extends ExactStorage { seqOfSeq.flatten.toSeq.groupBy(_._1).mapValues { seq => seq.map(_._2).toMap } } Await.result(future, 10 second) -// val (keyWithEq, reqJsLs) = toIncrementCountRequests(policy, counts).unzip(x => ((x._1, x._2), x._3)) -// -// val future = wsClient.url(s"$s2graphUrl/graphs/edges/incrementCount").post(Json.toJson(reqJsLs)).map { resp => -// resp.status match { -// case HttpStatus.SC_OK => -// val respSeq = resp.json.as[Seq[RespGraph]] -// -// val keyWithEqResult = { -// for { -// ((key, eq), RespGraph(success, result)) <- keyWithEq.zip(respSeq) -// } yield { -// (key, (eq, result)) -// } -// }.groupBy(_._1).mapValues{ seq => seq.map(_._2).toMap } -// keyWithEqResult -// case _ => -// throw new RuntimeException(s"update failed: $policy $counts") -// } -// } -// Await.result(future, 10 second) } def delete(policy: Counter, keys: Seq[ExactKeyTrait]): Unit = { @@ -201,7 +182,7 @@ case class ExactStorageGraph(config: Config) extends ExactStorage { val reqJs = Json.parse(reqJsStr) // log.warn(s"query: ${reqJs.toString()}") - wsClient.url(s"$s2graphUrl/graphs/getEdges").post(reqJs).map { resp => + wsClient.url(s"$s2graphReadOnlyUrl/graphs/getEdges").post(reqJs).map { resp => resp.status match { case HttpStatus.SC_OK => val respJs = resp.json @@ -261,7 +242,7 @@ case class ExactStorageGraph(config: Config) extends ExactStorage { val query = Json.obj("srcVertices" -> Json.arr(src), "steps" -> Json.arr(step)) // println(s"query: ${query.toString()}") - wsClient.url(s"$s2graphUrl/graphs/getEdges").post(query).map { resp => + wsClient.url(s"$s2graphReadOnlyUrl/graphs/getEdges").post(query).map { resp => resp.status match { case HttpStatus.SC_OK => val respJs = resp.json http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b159a7fb/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala ---------------------------------------------------------------------- diff --git a/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala b/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala index b0c0a41..9ada8d4 100644 --- a/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala +++ b/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala @@ -39,6 +39,7 @@ class RankingStorageGraph(config: Config) extends RankingStorage { private val labelPostfix = "_topK" val s2graphUrl = s2config.GRAPH_URL + val s2graphReadOnlyUrl = s2config.GRAPH_READONLY_URL val prepareCache = new CollectionCache[Option[Boolean]](CollectionCacheConfig(10000, 600)) val graphOp = new GraphOperation(config) @@ -246,7 +247,7 @@ class RankingStorageGraph(config: Config) extends RankingStorage { log.debug(strJs) val payload = Json.parse(strJs) - wsClient.url(s"$s2graphUrl/graphs/getEdges").post(payload).map { resp => + wsClient.url(s"$s2graphReadOnlyUrl/graphs/getEdges").post(payload).map { resp => resp.status match { case HttpStatus.SC_OK => (resp.json \ "results").asOpt[List[JsValue]].getOrElse(Nil) @@ -278,7 +279,7 @@ class RankingStorageGraph(config: Config) extends RankingStorage { ) ) - val future = wsClient.url(s"$s2graphUrl/graphs/checkEdges").post(checkReqJs).map { resp => + val future = wsClient.url(s"$s2graphReadOnlyUrl/graphs/checkEdges").post(checkReqJs).map { resp => resp.status match { case HttpStatus.SC_OK => val checkRespJs = resp.json http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b159a7fb/s2counter_loader/src/main/scala/s2/config/StreamingConfig.scala ---------------------------------------------------------------------- diff --git a/s2counter_loader/src/main/scala/s2/config/StreamingConfig.scala b/s2counter_loader/src/main/scala/s2/config/StreamingConfig.scala index 0880bdd..ba5e863 100644 --- a/s2counter_loader/src/main/scala/s2/config/StreamingConfig.scala +++ b/s2counter_loader/src/main/scala/s2/config/StreamingConfig.scala @@ -20,4 +20,5 @@ object StreamingConfig extends ConfigFunctions(S2ConfigFactory.config) { // graph url val GRAPH_URL = getOrElse("s2graph.url", "") + val GRAPH_READONLY_URL = getOrElse("s2graph.read-only.url", GRAPH_URL) } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b159a7fb/s2counter_loader/src/test/resources/application.conf ---------------------------------------------------------------------- diff --git a/s2counter_loader/src/test/resources/application.conf b/s2counter_loader/src/test/resources/application.conf index 369a6fa..02d68b2 100644 --- a/s2counter_loader/src/test/resources/application.conf +++ b/s2counter_loader/src/test/resources/application.conf @@ -75,3 +75,4 @@ profile.prefetch.size=100 # s2graph s2graph.url = "http://"${host}":9000"; +s2graph.real-only.url = ${s2graph.url}
