Minor. Add session information log when session is created (#325) * Add session information log when session is created
Change-Id: I8702ebe1d893cf328b4490e5b5d09f3afd02b7ce * add more logs Change-Id: I969edf4e680e59e3ecfbaa50431ae61bd03d4795 * simplify the code Change-Id: If2c7876ddb6020ace3872c9ae639d69d58b02f48 * Address the comments Change-Id: Iba5f10ad73ff67b97af8292769b254346b8ef7c8 Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/d51e9982 Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/d51e9982 Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/d51e9982 Branch: refs/heads/master Commit: d51e998204e3a10f9207280951f98edc8755c433 Parents: 61b206e Author: Saisai Shao <sai.sai.s...@gmail.com> Authored: Fri May 12 08:36:19 2017 +0800 Committer: Jeff Zhang <zjf...@gmail.com> Committed: Thu May 11 17:36:19 2017 -0700 ---------------------------------------------------------------------- .../cloudera/livy/server/batch/BatchSession.scala | 11 ++++++++--- .../livy/server/batch/CreateBatchRequest.scala | 17 +++++++++++++++++ .../interactive/CreateInteractiveRequest.scala | 18 ++++++++++++++++++ .../server/interactive/InteractiveSession.scala | 5 ++++- 4 files changed, 47 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d51e9982/server/src/main/scala/com/cloudera/livy/server/batch/BatchSession.scala ---------------------------------------------------------------------- diff --git a/server/src/main/scala/com/cloudera/livy/server/batch/BatchSession.scala b/server/src/main/scala/com/cloudera/livy/server/batch/BatchSession.scala index 452a9d8..1a097b7 100644 --- a/server/src/main/scala/com/cloudera/livy/server/batch/BatchSession.scala +++ b/server/src/main/scala/com/cloudera/livy/server/batch/BatchSession.scala @@ -25,7 +25,7 @@ import scala.util.Random import com.fasterxml.jackson.annotation.JsonIgnoreProperties -import com.cloudera.livy.LivyConf +import com.cloudera.livy.{LivyConf, Logging} import com.cloudera.livy.server.recovery.SessionStore import com.cloudera.livy.sessions.{Session, SessionState} import com.cloudera.livy.sessions.Session._ @@ -41,7 +41,7 @@ case class BatchRecoveryMetadata( version: Int = 1) extends RecoveryMetadata -object BatchSession { +object BatchSession extends Logging { val RECOVERY_SESSION_TYPE = "batch" def create( @@ -89,6 +89,8 @@ object BatchSession { SparkApp.create(appTag, None, Option(sparkSubmit), livyConf, Option(s)) } + info(s"Creating batch session $id: [owner: $owner, request: $request]") + new BatchSession( id, appTag, @@ -153,7 +155,10 @@ class BatchSession( synchronized { debug(s"$this state changed from $oldState to $newState") newState match { - case SparkApp.State.RUNNING => _state = SessionState.Running() + case SparkApp.State.RUNNING => + _state = SessionState.Running() + info(s"Batch session $id created [appid: ${appId.orNull}, state: ${state.toString}, " + + s"info: ${appInfo.asJavaMap}]") case SparkApp.State.FINISHED => _state = SessionState.Success() case SparkApp.State.KILLED | SparkApp.State.FAILED => _state = SessionState.Dead() http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d51e9982/server/src/main/scala/com/cloudera/livy/server/batch/CreateBatchRequest.scala ---------------------------------------------------------------------- diff --git a/server/src/main/scala/com/cloudera/livy/server/batch/CreateBatchRequest.scala b/server/src/main/scala/com/cloudera/livy/server/batch/CreateBatchRequest.scala index 99459d5..29c269c 100644 --- a/server/src/main/scala/com/cloudera/livy/server/batch/CreateBatchRequest.scala +++ b/server/src/main/scala/com/cloudera/livy/server/batch/CreateBatchRequest.scala @@ -37,4 +37,21 @@ class CreateBatchRequest { var name: Option[String] = None var conf: Map[String, String] = Map() + override def toString: String = { + s"[proxyUser: $proxyUser, " + + s"file: $file, " + + (if (args.nonEmpty) s"args: ${args.mkString(",")}, " else "") + + (if (jars.nonEmpty) s"jars: ${jars.mkString(",")}, " else "") + + (if (pyFiles.nonEmpty) s"pyFiles: ${pyFiles.mkString(",")}, " else "") + + (if (files.nonEmpty) s"files: ${files.mkString(",")}, " else "") + + (if (archives.nonEmpty) s"archives: ${archives.mkString(",")}, " else "") + + (if (driverMemory.isDefined) s"driverMemory: ${driverMemory.get}, " else "") + + (if (driverCores.isDefined) s"driverCores: ${driverCores.get}, " else "") + + (if (executorMemory.isDefined) s"executorMemory: ${executorMemory.get}, " else "") + + (if (executorCores.isDefined) s"executorCores: ${executorCores.get}, " else "") + + (if (numExecutors.isDefined) s"numExecutors: ${numExecutors.get}, " else "") + + (if (queue.isDefined) s"queue: ${queue.get}, " else "") + + (if (name.isDefined) s"name: ${name.get}, " else "") + + (if (conf.nonEmpty) s"conf: ${conf.mkString(",")}]" else "]") + } } http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d51e9982/server/src/main/scala/com/cloudera/livy/server/interactive/CreateInteractiveRequest.scala ---------------------------------------------------------------------- diff --git a/server/src/main/scala/com/cloudera/livy/server/interactive/CreateInteractiveRequest.scala b/server/src/main/scala/com/cloudera/livy/server/interactive/CreateInteractiveRequest.scala index a837527..7261199 100644 --- a/server/src/main/scala/com/cloudera/livy/server/interactive/CreateInteractiveRequest.scala +++ b/server/src/main/scala/com/cloudera/livy/server/interactive/CreateInteractiveRequest.scala @@ -36,4 +36,22 @@ class CreateInteractiveRequest { var name: Option[String] = None var conf: Map[String, String] = Map() var heartbeatTimeoutInSecond: Int = 0 + + override def toString: String = { + s"[kind: $kind, " + + s"proxyUser: $proxyUser, " + + (if (jars.nonEmpty) s"jars: ${jars.mkString(",")}, " else "") + + (if (pyFiles.nonEmpty) s"pyFiles: ${pyFiles.mkString(",")}, " else "") + + (if (files.nonEmpty) s"files: ${files.mkString(",")}, " else "") + + (if (archives.nonEmpty) s"archives: ${archives.mkString(",")}, " else "") + + (if (driverMemory.isDefined) s"driverMemory: ${driverMemory.get}, " else "") + + (if (driverCores.isDefined) s"driverCores: ${driverCores.get}, " else "") + + (if (executorMemory.isDefined) s"executorMemory: ${executorMemory.get}, " else "") + + (if (executorCores.isDefined) s"executorCores: ${executorCores.get}, " else "") + + (if (numExecutors.isDefined) s"numExecutors: ${numExecutors.get}, " else "") + + (if (queue.isDefined) s"queue: ${queue.get}, " else "") + + (if (name.isDefined) s"name: ${name.get}, " else "") + + (if (conf.nonEmpty) s"conf: ${conf.mkString(",")}, " else "") + + s"heartbeatTimeoutInSecond: $heartbeatTimeoutInSecond]" + } } http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d51e9982/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSession.scala ---------------------------------------------------------------------- diff --git a/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSession.scala b/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSession.scala index 6d10c57..05bc7be 100644 --- a/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSession.scala +++ b/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSession.scala @@ -98,7 +98,7 @@ object InteractiveSession extends Logging { builderProperties.getOrElseUpdate("spark.app.name", s"livy-session-$id") - info(s"Creating LivyClient for sessionId: $id") + info(s"Creating Interactive session $id: [owner: $owner, request: $request]") val builder = new LivyClientBuilder() .setAll(builderProperties.asJava) .setConf("livy.client.session-id", id.toString) @@ -421,6 +421,9 @@ class InteractiveSession( override def onJobSucceeded(job: JobHandle[Void], result: Void): Unit = { transition(SessionState.Running()) + info(s"Interactive session $id created [appid: ${appId.orNull}, owner: $owner, proxyUser:" + + s" $proxyUser, state: ${state.toString}, kind: ${kind.toString}, " + + s"info: ${appInfo.asJavaMap}]") } private def errorOut(): Unit = {