[S2GRAPH-117] migrated to play 2.5.9 - explicitly set to not using dependency injection - converted some Iteratees to Acumulators
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/fcb041b3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/fcb041b3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/fcb041b3 Branch: refs/heads/master Commit: fcb041b3cd14db591cd0cb868b118758ad70ad89 Parents: c7967ff Author: Jong Wook Kim <[email protected]> Authored: Mon Oct 10 00:13:12 2016 -0400 Committer: Jong Wook Kim <[email protected]> Committed: Mon Oct 10 01:44:29 2016 -0400 ---------------------------------------------------------------------- build.sbt | 2 +- project/Common.scala | 2 +- project/plugins.sbt | 2 +- .../counter/core/v2/ExactStorageGraph.scala | 6 ++++- .../counter/core/v2/GraphOperation.scala | 7 ++++- .../counter/core/v2/RankingStorageGraph.scala | 7 ++++- .../counter/loader/core/DimensionProps.scala | 6 ++++- .../controllers/ApplicationController.scala | 5 ++-- .../rest/play/controllers/JsonBodyParser.scala | 20 ++++++++------ .../org/apache/s2graph/rest/play/package.scala | 28 ++++++++++++++++++++ s2rest_play/build.sbt | 3 ++- 11 files changed, 70 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fcb041b3/build.sbt ---------------------------------------------------------------------- diff --git a/build.sbt b/build.sbt index 23ad8ef..d2df8a6 100755 --- a/build.sbt +++ b/build.sbt @@ -38,7 +38,7 @@ lazy val commonSettings = Seq( Revolver.settings -lazy val s2rest_play = project.enablePlugins(PlayScala) +lazy val s2rest_play = project.enablePlugins(PlayScala).disablePlugins(PlayLogback) .dependsOn(s2core, s2counter_core) .settings(commonSettings: _*) .settings(testOptions in Test += Tests.Argument("sequential")) http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fcb041b3/project/Common.scala ---------------------------------------------------------------------- diff --git a/project/Common.scala b/project/Common.scala index 0e92f05..fb4af84 100644 --- a/project/Common.scala +++ b/project/Common.scala @@ -21,7 +21,7 @@ import sbt._ object Common { val sparkVersion = "1.4.1" - val playVersion = "2.4.8" + val playVersion = "2.5.9" val hbaseVersion = "1.2.2" val hadoopVersion = "2.7.3" http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fcb041b3/project/plugins.sbt ---------------------------------------------------------------------- diff --git a/project/plugins.sbt b/project/plugins.sbt index 3efaa7d..a6e5381 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -19,7 +19,7 @@ // use the Play sbt plugin for Play projects -addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.4.8") +addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.5.9") // http://www.scalastyle.org/sbt.html addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.7.0") http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fcb041b3/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/ExactStorageGraph.scala ---------------------------------------------------------------------- diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/ExactStorageGraph.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/ExactStorageGraph.scala index 18cfeb2..caddf8d 100644 --- a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/ExactStorageGraph.scala +++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/ExactStorageGraph.scala @@ -19,6 +19,8 @@ package org.apache.s2graph.counter.core.v2 +import akka.actor.ActorSystem +import akka.stream.ActorMaterializer import com.typesafe.config.Config import org.apache.http.HttpStatus import org.apache.s2graph.core.mysqls.Label @@ -28,6 +30,7 @@ import org.apache.s2graph.counter.core.ExactCounter.ExactValueMap import org.apache.s2graph.counter.core._ import org.apache.s2graph.counter.models.Counter import org.apache.s2graph.counter.util.CartesianProduct +import org.asynchttpclient.DefaultAsyncHttpClientConfig import org.slf4j.LoggerFactory import play.api.libs.json._ import scala.concurrent.duration._ @@ -38,7 +41,8 @@ object ExactStorageGraph { implicit val respGraphFormat = Json.format[RespGraph] // using play-ws without play app - private val builder = new com.ning.http.client.AsyncHttpClientConfig.Builder() + implicit val materializer = ActorMaterializer.create(ActorSystem(getClass.getSimpleName)) + private val builder = new DefaultAsyncHttpClientConfig.Builder() private val wsClient = new play.api.libs.ws.ning.NingWSClient(builder.build) } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fcb041b3/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/GraphOperation.scala ---------------------------------------------------------------------- diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/GraphOperation.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/GraphOperation.scala index 29a509b..329f3d0 100644 --- a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/GraphOperation.scala +++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/GraphOperation.scala @@ -19,9 +19,13 @@ package org.apache.s2graph.counter.core.v2 +import akka.actor.ActorSystem +import akka.stream.ActorMaterializer import com.typesafe.config.Config import org.apache.http.HttpStatus import org.apache.s2graph.counter.config.S2CounterConfig +import org.apache.s2graph.counter.core.v2.ExactStorageGraph._ +import org.asynchttpclient.DefaultAsyncHttpClientConfig import org.slf4j.LoggerFactory import play.api.libs.json.{JsObject, JsValue, Json} import scala.concurrent.Await @@ -29,7 +33,8 @@ import scala.concurrent.duration._ class GraphOperation(config: Config) { // using play-ws without play app - private val builder = new com.ning.http.client.AsyncHttpClientConfig.Builder() + implicit val materializer = ActorMaterializer.create(ActorSystem(getClass.getSimpleName)) + private val builder = new DefaultAsyncHttpClientConfig.Builder() private val wsClient = new play.api.libs.ws.ning.NingWSClient(builder.build) private val s2config = new S2CounterConfig(config) val s2graphUrl = s2config.GRAPH_URL http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fcb041b3/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/RankingStorageGraph.scala ---------------------------------------------------------------------- diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/RankingStorageGraph.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/RankingStorageGraph.scala index 55800f2..b9ca9a4 100644 --- a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/RankingStorageGraph.scala +++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/RankingStorageGraph.scala @@ -19,15 +19,19 @@ package org.apache.s2graph.counter.core.v2 +import akka.actor.ActorSystem +import akka.stream.ActorMaterializer import com.typesafe.config.Config import org.apache.commons.httpclient.HttpStatus import org.apache.s2graph.core.GraphUtil import org.apache.s2graph.core.mysqls.Label import org.apache.s2graph.counter.config.S2CounterConfig import org.apache.s2graph.counter.core.RankingCounter.RankingValueMap +import org.apache.s2graph.counter.core.v2.ExactStorageGraph._ import org.apache.s2graph.counter.core.{RankingResult, RankingKey, RankingStorage} import org.apache.s2graph.counter.models.{Counter, CounterModel} import org.apache.s2graph.counter.util.{CollectionCacheConfig, CollectionCache} +import org.asynchttpclient.DefaultAsyncHttpClientConfig import org.slf4j.LoggerFactory import play.api.libs.json.{JsObject, JsString, JsValue, Json} import scala.concurrent.duration._ @@ -36,7 +40,8 @@ import scala.util.hashing.MurmurHash3 object RankingStorageGraph { // using play-ws without play app - private val builder = new com.ning.http.client.AsyncHttpClientConfig.Builder() + implicit val materializer = ActorMaterializer.create(ActorSystem(getClass.getSimpleName)) + private val builder = new DefaultAsyncHttpClientConfig.Builder() private val wsClient = new play.api.libs.ws.ning.NingWSClient(builder.build) } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fcb041b3/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/DimensionProps.scala ---------------------------------------------------------------------- diff --git a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/DimensionProps.scala b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/DimensionProps.scala index a1b9903..f941224 100644 --- a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/DimensionProps.scala +++ b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/DimensionProps.scala @@ -19,11 +19,14 @@ package org.apache.s2graph.counter.loader.core +import akka.actor.ActorSystem +import akka.stream.ActorMaterializer import org.apache.commons.httpclient.HttpStatus import org.apache.s2graph.core.mysqls.{Bucket, Experiment, Service} import org.apache.s2graph.counter.loader.config.StreamingConfig import org.apache.s2graph.counter.models.Counter import org.apache.s2graph.counter.util.{RetryAsync, CollectionCache, CollectionCacheConfig} +import org.asynchttpclient.DefaultAsyncHttpClientConfig import org.slf4j.LoggerFactory import play.api.libs.json._ import scala.annotation.tailrec @@ -34,7 +37,8 @@ import scala.util.Try object DimensionProps { // using play-ws without play app - private val builder = new com.ning.http.client.AsyncHttpClientConfig.Builder() + implicit val materializer = ActorMaterializer.create(ActorSystem(getClass.getSimpleName)) + private val builder = new DefaultAsyncHttpClientConfig.Builder() private val client = new play.api.libs.ws.ning.NingWSClient(builder.build) private val log = LoggerFactory.getLogger(this.getClass) http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fcb041b3/s2rest_play/app/org/apache/s2graph/rest/play/controllers/ApplicationController.scala ---------------------------------------------------------------------- diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/ApplicationController.scala b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/ApplicationController.scala index 4170eec..707b98c 100644 --- a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/ApplicationController.scala +++ b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/ApplicationController.scala @@ -19,9 +19,11 @@ package org.apache.s2graph.rest.play.controllers +import akka.util.ByteString import org.apache.s2graph.core.GraphExceptions.BadQueryException import org.apache.s2graph.core.PostProcess import org.apache.s2graph.core.utils.logger +import play.api.http.HttpEntity import play.api.libs.iteratee.Enumerator import play.api.libs.json.{JsString, JsValue} import play.api.mvc._ @@ -69,8 +71,7 @@ object ApplicationController extends Controller { } else { Result( header = ResponseHeader(OK), - body = Enumerator(json.toString.getBytes()), - connection = HttpConnection.Close + body = HttpEntity.Strict(ByteString(json.toString.getBytes()), Some(applicationJsonHeader)) ).as(applicationJsonHeader).withHeaders((CONNECTION -> "close") +: headers: _*) } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fcb041b3/s2rest_play/app/org/apache/s2graph/rest/play/controllers/JsonBodyParser.scala ---------------------------------------------------------------------- diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/JsonBodyParser.scala b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/JsonBodyParser.scala index 3245bb6..aa507d9 100644 --- a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/JsonBodyParser.scala +++ b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/JsonBodyParser.scala @@ -19,10 +19,12 @@ package org.apache.s2graph.rest.play.controllers +import akka.util.ByteString import org.apache.s2graph.core.utils.logger import play.api.Play import play.api.libs.iteratee.Iteratee import play.api.libs.json.{JsValue, Json} +import play.api.libs.streams.Streams import play.api.mvc._ import scala.concurrent.Future @@ -50,9 +52,11 @@ object s2parse extends BodyParsers { import play.api.libs.iteratee.Execution.Implicits.trampoline import play.api.libs.iteratee.Traversable - Traversable.takeUpTo[Array[Byte]](maxLength) - .transform(Iteratee.consume[Array[Byte]]().map(c => new String(c, "UTF-8"))) + val iteratee = Traversable.takeUpTo[ByteString](maxLength) + .transform(Iteratee.consume[ByteString]().map(_.utf8String)) .flatMap(Iteratee.eofOrElse(Results.EntityTooLarge)) + + Streams.iterateeToAccumulator(iteratee) } def json(maxLength: Int): BodyParser[JsValue] = when( @@ -76,14 +80,14 @@ object s2parse extends BodyParsers { import scala.util.control.Exception._ - val bodyParser: Iteratee[Array[Byte], Either[Result, Either[Future[Result], A]]] = - Traversable.takeUpTo[Array[Byte]](maxLength).transform( - Iteratee.consume[Array[Byte]]().map { bytes => + val bodyParser: Iteratee[ByteString, Either[Result, Either[Future[Result], A]]] = + Traversable.takeUpTo[ByteString](maxLength).transform( + Iteratee.consume[ByteString]().map { bytes => allCatch[A].either { - parser(request, bytes) + parser(request, bytes.toByteBuffer.array()) }.left.map { case NonFatal(e) => - val txt = new String(bytes) + val txt = bytes.utf8String logger.error(s"$errorMessage: $txt", e) createBadResult(s"$errorMessage: $e")(request) case t => throw t @@ -91,7 +95,7 @@ object s2parse extends BodyParsers { } ).flatMap(Iteratee.eofOrElse(Results.EntityTooLarge)) - bodyParser.mapM { + Streams.iterateeToAccumulator(bodyParser).mapFuture { case Left(tooLarge) => Future.successful(Left(tooLarge)) case Right(Left(badResult)) => badResult.map(Left.apply) case Right(Right(body)) => Future.successful(Right(body)) http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fcb041b3/s2rest_play/app/org/apache/s2graph/rest/play/package.scala ---------------------------------------------------------------------- diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/package.scala b/s2rest_play/app/org/apache/s2graph/rest/play/package.scala new file mode 100644 index 0000000..be4175a --- /dev/null +++ b/s2rest_play/app/org/apache/s2graph/rest/play/package.scala @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.s2graph.rest + +import akka.actor.ActorSystem +import akka.stream.ActorMaterializer + +package object play { + implicit val actorSystem = ActorSystem("s2graph") + implicit val materializer = ActorMaterializer.create(actorSystem) +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fcb041b3/s2rest_play/build.sbt ---------------------------------------------------------------------- diff --git a/s2rest_play/build.sbt b/s2rest_play/build.sbt index 01f4c85..77e13d4 100644 --- a/s2rest_play/build.sbt +++ b/s2rest_play/build.sbt @@ -27,5 +27,6 @@ libraryDependencies := loggingRuntime ++ (libraryDependencies.value ++ Seq(ws, f "org.specs2" %% "specs2-core" % "3.8.5" % "test" ) -enablePlugins(JavaServerAppPackaging) +routesGenerator := StaticRoutesGenerator +enablePlugins(JavaServerAppPackaging)
