[S2GRAPH-117] migrated to play 2.4.8 (still emitting logback error)
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/c7967ff3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/c7967ff3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/c7967ff3 Branch: refs/heads/master Commit: c7967ff346e40ed6c27fe570931cc6defd87ffe8 Parents: 1f5cfde Author: Jong Wook Kim <[email protected]> Authored: Sun Oct 9 23:25:43 2016 -0400 Committer: Jong Wook Kim <[email protected]> Committed: Mon Oct 10 01:44:15 2016 -0400 ---------------------------------------------------------------------- project/Common.scala | 5 +- project/plugins.sbt | 2 +- .../org/apache/s2graph/core/PostProcess.scala | 1 - .../s2graph/core/rest/RequestParser.scala | 68 +++++++++++--------- .../counter/core/v2/ExactStorageGraph.scala | 4 +- .../loader/core/CounterEtlFunctions.scala | 2 +- .../counter/loader/core/CounterEtlItem.scala | 8 +-- .../counter/loader/core/CounterFunctions.scala | 2 +- .../rest/play/controllers/EdgeController.scala | 4 +- s2rest_play/build.sbt | 5 +- 10 files changed, 56 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c7967ff3/project/Common.scala ---------------------------------------------------------------------- diff --git a/project/Common.scala b/project/Common.scala index f0fb222..0e92f05 100644 --- a/project/Common.scala +++ b/project/Common.scala @@ -21,7 +21,7 @@ import sbt._ object Common { val sparkVersion = "1.4.1" - val playVersion = "2.5.9" + val playVersion = "2.4.8" val hbaseVersion = "1.2.2" val hadoopVersion = "2.7.3" @@ -38,7 +38,8 @@ object Common { val loggingExcludes = Seq( ExclusionRule("commons-logging", "commons-logging"), ExclusionRule("log4j", "log4j"), - ExclusionRule("ch.qos.logback", "*"), + ExclusionRule("ch.qos.logback", "logback-classic"), + ExclusionRule("ch.qos.logback", "logback-core"), ExclusionRule("org.slf4j", "jcl-over-slf4j"), ExclusionRule("org.slf4j", "log4j-over-slf4j"), ExclusionRule("org.slf4j", "slf4j-log4j12"), http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c7967ff3/project/plugins.sbt ---------------------------------------------------------------------- diff --git a/project/plugins.sbt b/project/plugins.sbt index d98d1cb..3efaa7d 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -19,7 +19,7 @@ // use the Play sbt plugin for Play projects -addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.3.10") +addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.4.8") // http://www.scalastyle.org/sbt.html addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.7.0") http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c7967ff3/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala b/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala index caaa408..923eeef 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala @@ -186,7 +186,6 @@ object PostProcess extends JSONParser { private def buildReplaceJson(jsValue: JsValue)(mapper: JsValue => JsValue): JsValue = { def traverse(js: JsValue): JsValue = js match { case JsNull => mapper(JsNull) - case JsUndefined() => mapper(JsUndefined("")) case JsNumber(v) => mapper(js) case JsString(v) => mapper(js) case JsBoolean(v) => mapper(js) http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c7967ff3/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala b/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala index 61db2ef..f67a89b 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala @@ -29,6 +29,7 @@ import org.apache.s2graph.core.mysqls._ import org.apache.s2graph.core.parsers.{Where, WhereParser} import org.apache.s2graph.core.types._ import play.api.libs.json._ +import play.api.libs.json.Reads._ import scala.util.{Failure, Success, Try} @@ -92,7 +93,7 @@ class RequestParser(config: Config) extends JSONParser { private def extractScoring(labelId: Int, value: JsValue) = { val ret = for { - js <- parse[Option[JsObject]](value, "scoring") + js <- parseOption[JsObject](value, "scoring") } yield { for { (k, v) <- js.fields @@ -113,16 +114,16 @@ class RequestParser(config: Config) extends JSONParser { val jsValue = Json.parse(replaced) def extractKv(js: JsValue) = js match { - case JsObject(obj) => obj + case JsObject(map) => map.toSeq case JsArray(arr) => arr.flatMap { - case JsObject(obj) => obj - case _ => throw new RuntimeException(s"cannot support json type $js") + case JsObject(map) => map.toSeq + case _ => throw new RuntimeException(s"cannot support json type: $js") } case _ => throw new RuntimeException(s"cannot support json type: $js") } val ret = for { - js <- parse[Option[JsObject]](jsValue, "interval") + js <- parseOption[JsObject](jsValue, "interval") fromJs <- (js \ "from").asOpt[JsValue] toJs <- (js \ "to").asOpt[JsValue] } yield { @@ -139,10 +140,10 @@ class RequestParser(config: Config) extends JSONParser { val jsValue = Json.parse(replaced) for { - js <- parse[Option[JsObject]](jsValue, "duration") + js <- parseOption[JsObject](jsValue, "duration") } yield { - val minTs = parse[Option[Long]](js, "from").getOrElse(Long.MaxValue) - val maxTs = parse[Option[Long]](js, "to").getOrElse(Long.MinValue) + val minTs = parseOption[Long](js, "from").getOrElse(Long.MaxValue) + val maxTs = parseOption[Long](js, "to").getOrElse(Long.MinValue) if (minTs > maxTs) { throw new BadQueryException("Duration error. Timestamp of From cannot be larger than To.") @@ -154,7 +155,7 @@ class RequestParser(config: Config) extends JSONParser { def extractHas(label: Label, jsValue: JsValue) = { val ret = for { - js <- parse[Option[JsObject]](jsValue, "has") + js <- parseOption[JsObject](jsValue, "has") } yield { for { (k, v) <- js.fields @@ -346,12 +347,12 @@ class RequestParser(config: Config) extends JSONParser { private def parseQueryParam(labelGroup: JsValue): Option[QueryParam] = { for { - labelName <- parse[Option[String]](labelGroup, "label") + labelName <- parseOption[String](labelGroup, "label") } yield { val label = Label.findByName(labelName).getOrElse(throw BadQueryException(s"$labelName not found")) - val direction = parse[Option[String]](labelGroup, "direction").map(GraphUtil.toDirection(_)).getOrElse(0) + val direction = parseOption[String](labelGroup, "direction").map(GraphUtil.toDirection(_)).getOrElse(0) val limit = { - parse[Option[Int]](labelGroup, "limit") match { + parseOption[Int](labelGroup, "limit") match { case None => defaultLimit case Some(l) if l < 0 => maxLimit case Some(l) if l >= 0 => @@ -359,12 +360,12 @@ class RequestParser(config: Config) extends JSONParser { Math.min(l, default) } } - val offset = parse[Option[Int]](labelGroup, "offset").getOrElse(0) + val offset = parseOption[Int](labelGroup, "offset").getOrElse(0) val interval = extractInterval(label, labelGroup) val duration = extractDuration(label, labelGroup) val scoring = extractScoring(label.id.get, labelGroup).getOrElse(List.empty[(Byte, Double)]).toList - val exclude = parse[Option[Boolean]](labelGroup, "exclude").getOrElse(false) - val include = parse[Option[Boolean]](labelGroup, "include").getOrElse(false) + val exclude = parseOption[Boolean](labelGroup, "exclude").getOrElse(false) + val include = parseOption[Boolean](labelGroup, "include").getOrElse(false) val hasFilter = extractHas(label, labelGroup) val labelWithDir = LabelWithDirection(label.id.get, direction) val indexNameOpt = (labelGroup \ "index").asOpt[String] @@ -392,7 +393,7 @@ class RequestParser(config: Config) extends JSONParser { } val threshold = (labelGroup \ "threshold").asOpt[Double].getOrElse(QueryParam.DefaultThreshold) // TODO: refactor this. dirty - val duplicate = parse[Option[String]](labelGroup, "duplicate").map(s => Query.DuplicatePolicy(s)) + val duplicate = parseOption[String](labelGroup, "duplicate").map(s => Query.DuplicatePolicy(s)) val outputField = (labelGroup \ "outputField").asOpt[String].map(s => Json.arr(Json.arr(s))) val transformer = if (outputField.isDefined) outputField else (labelGroup \ "transform").asOpt[JsValue] @@ -430,16 +431,23 @@ class RequestParser(config: Config) extends JSONParser { } private def parse[R](js: JsValue, key: String)(implicit read: Reads[R]): R = { - (js \ key).validate[R] - .fold( - errors => { - val msg = (JsError.toFlatJson(errors) \ "obj").as[List[JsValue]].map(x => x \ "msg") - val e = Json.obj("args" -> key, "error" -> msg) - throw new GraphExceptions.JsonParseException(Json.obj("error" -> key).toString) - }, - r => { - r - }) + (js \ key).validate[R] match { + case JsError(errors) => + val msg = (JsError.toFlatJson(errors) \ "obj").as[List[JsValue]].flatMap(x => (x \ "msg").toOption) + val e = Json.obj("args" -> key, "error" -> msg) + throw new GraphExceptions.JsonParseException(Json.obj("error" -> key).toString) + case JsSuccess(result, _) => result + } + } + + private def parseOption[R](js: JsValue, key: String)(implicit read: Reads[R]): Option[R] = { + (js \ key).validateOpt[R] match { + case JsError(errors) => + val msg = (JsError.toFlatJson(errors) \ "obj").as[List[JsValue]].flatMap(x => (x \ "msg").toOption) + val e = Json.obj("args" -> key, "error" -> msg) + throw new GraphExceptions.JsonParseException(Json.obj("error" -> key).toString) + case JsSuccess(result, _) => result + } } def toJsValues(jsValue: JsValue): List[JsValue] = { @@ -478,7 +486,7 @@ class RequestParser(config: Config) extends JSONParser { val label = parse[String](jsValue, "label") val timestamp = parse[Long](jsValue, "timestamp") - val direction = parse[Option[String]](jsValue, "direction").getOrElse("") + val direction = parseOption[String](jsValue, "direction").getOrElse("") val props = (jsValue \ "props").asOpt[JsValue].getOrElse("{}") for { srcId <- srcIds @@ -494,7 +502,7 @@ class RequestParser(config: Config) extends JSONParser { def toVertex(jsValue: JsValue, operation: String, serviceName: Option[String] = None, columnName: Option[String] = None): Vertex = { val id = parse[JsValue](jsValue, "id") - val ts = parse[Option[Long]](jsValue, "timestamp").getOrElse(System.currentTimeMillis()) + val ts = parseOption[Long](jsValue, "timestamp").getOrElse(System.currentTimeMillis()) val sName = if (serviceName.isEmpty) parse[String](jsValue, "serviceName") else serviceName.get val cName = if (columnName.isEmpty) parse[String](jsValue, "columnName") else columnName.get val props = (jsValue \ "props").asOpt[JsObject].getOrElse(Json.obj()) @@ -511,7 +519,7 @@ class RequestParser(config: Config) extends JSONParser { Prop(propName, defaultValue, dataType) } - def toPropsElements(jsValue: JsValue): Seq[Prop] = for { + def toPropsElements(jsValue: JsLookupResult): Seq[Prop] = for { jsObj <- jsValue.asOpt[Seq[JsValue]].getOrElse(Nil) } yield { val propName = (jsObj \ "name").as[String] @@ -523,7 +531,7 @@ class RequestParser(config: Config) extends JSONParser { Prop(propName, defaultValue, dataType) } - def toIndicesElements(jsValue: JsValue): Seq[Index] = for { + def toIndicesElements(jsValue: JsLookupResult): Seq[Index] = for { jsObj <- jsValue.as[Seq[JsValue]] indexName = (jsObj \ "name").as[String] propNames = (jsObj \ "propNames").as[Seq[String]] http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c7967ff3/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/ExactStorageGraph.scala ---------------------------------------------------------------------- diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/ExactStorageGraph.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/ExactStorageGraph.scala index 3a196ef..18cfeb2 100644 --- a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/ExactStorageGraph.scala +++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/ExactStorageGraph.scala @@ -209,10 +209,10 @@ case class ExactStorageGraph(config: Config) extends ExactStorage { } private def resultToExactKeyValues(policy: Counter, result: JsValue): (ExactKeyTrait, (core.ExactQualifier, Long)) = { - val from = result \ "from" match { + val from = (result \ "from").get match { case s: JsString => s.as[String] case n: JsNumber => n.as[Long].toString - case x: JsValue => throw new RuntimeException(s"$x's type must be string or number") + case x => throw new RuntimeException(s"$x's type must be string or number") } val dimension = (result \ "to").as[String] val props = result \ "props" http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c7967ff3/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctions.scala ---------------------------------------------------------------------- diff --git a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctions.scala b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctions.scala index 7dcf48a..247cd07 100644 --- a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctions.scala +++ b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctions.scala @@ -69,7 +69,7 @@ object CounterEtlFunctions extends Logging { } yield { val jsValue = variable match { case "_from" => JsString(srcId) - case s => dimension \ s + case s => (dimension \ s).get } s"[[$variable]]" -> jsValue } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c7967ff3/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlItem.scala ---------------------------------------------------------------------- diff --git a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlItem.scala b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlItem.scala index e3bb884..6eaa6e2 100644 --- a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlItem.scala +++ b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlItem.scala @@ -30,10 +30,10 @@ case class CounterEtlItem(ts: Long, service: String, action: String, item: Strin } lazy val value = { - property \ "value" match { - case JsNumber(n) => n.longValue() - case JsString(s) => s.toLong - case _: JsUndefined => 1L + (property \ "value").toOption match { + case Some(JsNumber(n)) => n.longValue() + case Some(JsString(s)) => s.toLong + case None => 1L case _ => throw new Exception("wrong type") } } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c7967ff3/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterFunctions.scala ---------------------------------------------------------------------- diff --git a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterFunctions.scala b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterFunctions.scala index 05423b1..7b272be 100644 --- a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterFunctions.scala +++ b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterFunctions.scala @@ -72,7 +72,7 @@ object CounterFunctions extends Logging with WithKafka { Try { for { k <- keys - jsValue = dimension \ k + jsValue <- (dimension \ k).toOption } yield { jsValue match { case JsNumber(n) => n.toString() http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c7967ff3/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala ---------------------------------------------------------------------- diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala index 478df99..ec8324c 100644 --- a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala +++ b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala @@ -40,10 +40,12 @@ object EdgeController extends Controller { private val s2: Graph = org.apache.s2graph.rest.play.Global.s2graph private val requestParser: RequestParser = org.apache.s2graph.rest.play.Global.s2parser + private def jsToStr(js: JsValue): String = js match { case JsString(s) => s - case _ => js.toString() + case obj => obj.toString() } + private def jsToStr(js: JsLookupResult): String = js.toOption.map(jsToStr).getOrElse("undefined") def toTsv(jsValue: JsValue, op: String): String = { val ts = jsToStr(jsValue \ "timestamp") http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c7967ff3/s2rest_play/build.sbt ---------------------------------------------------------------------- diff --git a/s2rest_play/build.sbt b/s2rest_play/build.sbt index dfe7205..01f4c85 100644 --- a/s2rest_play/build.sbt +++ b/s2rest_play/build.sbt @@ -22,8 +22,9 @@ name := "s2rest_play" scalacOptions in Test ++= Seq("-Yrangepos") -libraryDependencies := loggingRuntime ++ (libraryDependencies.value ++ Seq(ws, filters)).map(_.excludeLogging()) ++ Seq( - "com.google.guava" % "guava" % "12.0.1" force() // use this old version of guava to avoid incompatibility +libraryDependencies := loggingRuntime ++ (libraryDependencies.value ++ Seq(ws, filters, specs2 % Test)).map(_.excludeLogging()) ++ Seq( + "com.google.guava" % "guava" % "12.0.1" force(), // use this old version of guava to avoid incompatibility + "org.specs2" %% "specs2-core" % "3.8.5" % "test" ) enablePlugins(JavaServerAppPackaging)
